diff --git a/DOCKER_README.md b/DOCKER_README.md index cbe977a39ac..516c6ac66c2 100644 --- a/DOCKER_README.md +++ b/DOCKER_README.md @@ -59,7 +59,7 @@ docker run -d \ -v :/config \ -p 47984-47990:47984-47990/tcp \ -p 48010:48010 \ - -p 47998-48000:47998-48000/udp \ + -p 47998-48001:47998-48001/udp \ ``` @@ -83,7 +83,7 @@ services: ports: - "47984-47990:47984-47990/tcp" - "48010:48010" - - "47998-48000:47998-48000/udp" + - "47998-48001:47998-48001/udp" ``` ### Using podman run @@ -101,7 +101,7 @@ podman run -d \ -v :/config \ -p 47984-47990:47984-47990/tcp \ -p 48010:48010 \ - -p 47998-48000:47998-48000/udp \ + -p 47998-48001:47998-48001/udp \ ``` diff --git a/docker/archlinux.dockerfile b/docker/archlinux.dockerfile index 89071f60245..5a07fa4d1df 100644 --- a/docker/archlinux.dockerfile +++ b/docker/archlinux.dockerfile @@ -132,7 +132,7 @@ _INSTALL_SUNSHINE # network setup EXPOSE 47984-47990/tcp EXPOSE 48010 -EXPOSE 47998-48000/udp +EXPOSE 47998-48001/udp # setup user ARG PGID=1000 diff --git a/docker/debian-bookworm.dockerfile b/docker/debian-bookworm.dockerfile index f88cea0534c..d36d6f556a9 100644 --- a/docker/debian-bookworm.dockerfile +++ b/docker/debian-bookworm.dockerfile @@ -76,7 +76,7 @@ _INSTALL_SUNSHINE # network setup EXPOSE 47984-47990/tcp EXPOSE 48010 -EXPOSE 47998-48000/udp +EXPOSE 47998-48001/udp # setup user ARG PGID=1000 diff --git a/docker/ubuntu-22.04.dockerfile b/docker/ubuntu-22.04.dockerfile index 7bb24eb8bb7..2cc3ca33284 100644 --- a/docker/ubuntu-22.04.dockerfile +++ b/docker/ubuntu-22.04.dockerfile @@ -76,7 +76,7 @@ _INSTALL_SUNSHINE # network setup EXPOSE 47984-47990/tcp EXPOSE 48010 -EXPOSE 47998-48000/udp +EXPOSE 47998-48001/udp # setup user ARG PGID=1000 diff --git a/docker/ubuntu-24.04.dockerfile b/docker/ubuntu-24.04.dockerfile index 984cfdcb157..e28ef9ac342 100644 --- a/docker/ubuntu-24.04.dockerfile +++ b/docker/ubuntu-24.04.dockerfile @@ -76,7 +76,7 @@ _INSTALL_SUNSHINE # network setup EXPOSE 47984-47990/tcp EXPOSE 48010 -EXPOSE 47998-48000/udp +EXPOSE 47998-48001/udp # setup user ARG PGID=1001 diff --git a/src/audio.cpp b/src/audio.cpp index 0d287071a25..dc915824115 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -11,9 +11,11 @@ // local includes #include "audio.h" #include "config.h" +#include "crypto.h" #include "globals.h" #include "logging.h" #include "platform/common.h" +#include "stream.h" #include "thread_safe.h" #include "utility.h" @@ -83,7 +85,7 @@ namespace audio { }, }; - void encodeThread(sample_queue_t samples, config_t config, void *channel_data) { + void encodeThread(sample_queue_t samples, const config_t& config, void *channel_data) { auto packets = mail::man->queue(mail::audio_packets); auto stream = stream_configs[map_stream(config.channels, config.flags[config_t::HIGH_QUALITY])]; if (config.flags[config_t::CUSTOM_SURROUND_PARAMS]) { @@ -298,6 +300,10 @@ namespace audio { ctx.sink = std::move(*sink); + // Create virtual microphone for lobby-style chat + BOOST_LOG(info) << "Setting up virtual microphone for lobby chat"; + ctx.control->create_virtual_microphone("sunshine-virtual-mic"); + fg.disable(); return 0; } @@ -322,4 +328,299 @@ namespace audio { stream.coupledStreams = params.coupledStreams; stream.mapping = params.mapping; } + + // Multi-client microphone session management + struct mic_client_session_t { + uint32_t client_id; + std::unique_ptr opus_decoder; + std::unordered_map> audio_streams; + std::chrono::steady_clock::time_point last_activity; + + // Packet sequencing and timing + std::unordered_map expected_sequence; // stream_id -> next expected sequence + std::unordered_map last_timestamp; // stream_id -> last received timestamp + std::unordered_map stream_start_time; // stream_id -> start time + }; + + static std::unordered_map> mic_clients; + static std::mutex mic_clients_mutex; + + void mic_receive(safe::mail_t mail, const config_t& config, void *channel_data, stream::session_t *session) { + if (!config::audio.enable_mic_passthrough) { + BOOST_LOG(warning) << "Microphone pass-through requested but disabled in config"sv; + return; + } + + auto shutdown_event = mail->event(mail::shutdown); + auto packets = mail->queue(mail::mic_packets); + + // Create microphone output device + auto audio_ctx = get_audio_ctx_ref(); + if (!audio_ctx || !audio_ctx->control) { + BOOST_LOG(error) << "No audio control context available for microphone output"sv; + return; + } + + const std::string &mic_sink = config::audio.mic_sink.empty() ? "default" : config::audio.mic_sink; + auto mic_output = audio_ctx->control->mic_output(1, 48000, mic_sink); + if (!mic_output) { + BOOST_LOG(error) << "Failed to initialize microphone output device: "sv << mic_sink; + return; + } + + if (mic_output->start()) { + BOOST_LOG(error) << "Failed to start microphone output device"sv; + return; + } + + // Create virtual microphone output for lobby-style chat + auto virtual_mic_output = audio_ctx->control->mic_output(1, 48000, "sunshine-virtual-mic"); + bool virtual_mic_available = false; + if (virtual_mic_output && virtual_mic_output->start() == 0) { + virtual_mic_available = true; + BOOST_LOG(info) << "Virtual microphone output enabled for lobby chat"; + } else { + BOOST_LOG(warning) << "Virtual microphone output not available - lobby chat disabled"; + } + + BOOST_LOG(info) << "Started microphone receiver thread"sv; + + auto opus_dec = opus_decoder_create(48000, 1, nullptr); + if (!opus_dec) { + BOOST_LOG(error) << "Failed to create Opus decoder for microphone"sv; + return; + } + + std::vector audio_decode_buffer(960); // 20ms at 48kHz mono + + while (auto packet = packets->pop()) { + if (shutdown_event->peek()) { + break; + } + + // Parse microphone packet header for client identification + auto packet_data = packet->second.data(); + auto packet_size = packet->second.size(); + + if (packet_size < sizeof(stream::mic_packet_header_t)) { + BOOST_LOG(warning) << "Received undersized microphone packet: " << packet_size << " bytes"sv; + continue; + } + + auto header = reinterpret_cast(packet_data); + + // Validate packet header + if (header->version != stream::MIC_PROTOCOL_VERSION) { + BOOST_LOG(warning) << "Unsupported microphone protocol version: " << header->version; + continue; + } + + if (header->packet_type != stream::MIC_PACKET_AUDIO) { + BOOST_LOG(debug) << "Skipping non-audio microphone packet type: " << header->packet_type; + continue; + } + + if (header->payload_size + sizeof(stream::mic_packet_header_t) != packet_size) { + BOOST_LOG(warning) << "Microphone packet size mismatch: expected " + << header->payload_size + sizeof(stream::mic_packet_header_t) + << ", got " << packet_size; + continue; + } + + // Log client and stream info for debugging + BOOST_LOG(debug) << "Processing microphone packet from client " << header->client_id + << ", stream " << header->stream_id + << ", sequence " << header->sequence + << ", timestamp " << header->timestamp + << ", encrypted " << ((header->flags & stream::MIC_FLAG_ENCRYPTED) ? "yes" : "no"); + + // Handle encrypted packets + std::vector decrypted_payload; + const unsigned char* opus_data; + size_t opus_size; + + if (header->flags & stream::MIC_FLAG_ENCRYPTED) { + // Check if session has encryption enabled + if (!session || !session->microphone.cipher) { + BOOST_LOG(warning) << "Received encrypted microphone packet but encryption not enabled for session"; + continue; + } + + // Construct IV using NIST SP 800-38D deterministic method + crypto::aes_t iv(12); + std::copy_n((uint8_t *) &header->sequence, sizeof(header->sequence), std::begin(iv)); + iv[10] = 'M'; // Microphone + iv[11] = 'C'; // Client originated + + // Decrypt the payload + auto encrypted_payload = packet_data + sizeof(stream::mic_packet_header_t); + std::string_view ciphertext_with_tag{ + reinterpret_cast(header->tag), + sizeof(header->tag) + header->payload_size + }; + + if (session->microphone.cipher->decrypt(ciphertext_with_tag, decrypted_payload, &iv)) { + BOOST_LOG(warning) << "Failed to decrypt microphone packet from client " << header->client_id; + continue; + } + + opus_data = decrypted_payload.data(); + opus_size = decrypted_payload.size(); + } else { + // Unencrypted packet - use payload directly + opus_data = reinterpret_cast(packet_data + sizeof(stream::mic_packet_header_t)); + opus_size = header->payload_size; + } + + // Get or create client session + std::lock_guard lock(mic_clients_mutex); + + auto client_it = mic_clients.find(header->client_id); + if (client_it == mic_clients.end()) { + // Check client limit (hardcoded to 4 for now, will be made configurable per client request) + constexpr uint32_t MAX_MIC_CLIENTS = 4; + if (mic_clients.size() >= MAX_MIC_CLIENTS) { + BOOST_LOG(warning) << "Microphone client limit reached (" << MAX_MIC_CLIENTS + << "), rejecting client " << header->client_id; + continue; + } + + // Create new client session + auto client_session = std::make_unique(); + client_session->client_id = header->client_id; + client_session->opus_decoder = std::unique_ptr( + reinterpret_cast(opus_decoder_create(48000, 1, nullptr)) + ); + + if (!client_session->opus_decoder) { + BOOST_LOG(error) << "Failed to create Opus decoder for client " << header->client_id; + continue; + } + + client_it = mic_clients.emplace(header->client_id, std::move(client_session)).first; + BOOST_LOG(info) << "Created new microphone session for client " << header->client_id; + } + + auto& client_session = client_it->second; + client_session->last_activity = std::chrono::steady_clock::now(); + + // Validate packet sequence and timestamp for this stream + auto stream_id = header->stream_id; + auto expected_seq_it = client_session->expected_sequence.find(stream_id); + + if (expected_seq_it != client_session->expected_sequence.end()) { + // Check sequence number for existing stream + uint16_t expected_seq = expected_seq_it->second; + if (header->sequence != expected_seq) { + if (header->sequence < expected_seq) { + // Duplicate or out-of-order packet - ignore + BOOST_LOG(debug) << "Ignoring duplicate/old packet from client " << header->client_id + << ", stream " << stream_id + << ", got sequence " << header->sequence + << ", expected " << expected_seq; + continue; + } else { + // Missing packets detected + uint16_t missed_packets = header->sequence - expected_seq; + BOOST_LOG(warning) << "Missed " << missed_packets << " packet(s) from client " + << header->client_id << ", stream " << stream_id + << ", expected " << expected_seq << ", got " << header->sequence; + } + } + + // Validate timestamp progression (should be monotonically increasing) + auto last_timestamp_it = client_session->last_timestamp.find(stream_id); + if (last_timestamp_it != client_session->last_timestamp.end()) { + uint32_t last_ts = last_timestamp_it->second; + if (header->timestamp <= last_ts) { + BOOST_LOG(warning) << "Non-monotonic timestamp from client " << header->client_id + << ", stream " << stream_id + << ", current " << header->timestamp + << ", last " << last_ts; + } + } + } else { + // First packet for this stream - initialize tracking + BOOST_LOG(info) << "Starting sequence tracking for client " << header->client_id + << ", stream " << stream_id + << ", starting sequence " << header->sequence; + client_session->stream_start_time[stream_id] = std::chrono::steady_clock::now(); + } + + // Update sequence and timestamp tracking + client_session->expected_sequence[stream_id] = header->sequence + 1; + client_session->last_timestamp[stream_id] = header->timestamp; + + // Get or create audio stream for this client/stream combination + auto stream_it = client_session->audio_streams.find(header->stream_id); + if (stream_it == client_session->audio_streams.end()) { + // Check stream limit per client (hardcoded to 2 for now, will be made configurable per client request) + constexpr uint32_t MAX_STREAMS_PER_CLIENT = 2; + if (client_session->audio_streams.size() >= MAX_STREAMS_PER_CLIENT) { + BOOST_LOG(warning) << "Stream limit reached for client " << header->client_id + << " (" << MAX_STREAMS_PER_CLIENT << "), ignoring stream " << header->stream_id; + continue; + } + + // Create new audio output stream + auto client_audio_ctx = get_audio_ctx_ref(); + if (!client_audio_ctx || !client_audio_ctx->control) { + BOOST_LOG(error) << "No audio control context available for client " << header->client_id; + continue; + } + + const std::string client_mic_sink = config::audio.mic_sink.empty() ? "default" : config::audio.mic_sink; + auto client_mic_output = client_audio_ctx->control->mic_output(1, 48000, client_mic_sink); + if (!client_mic_output || client_mic_output->start()) { + BOOST_LOG(error) << "Failed to create audio output for client " << header->client_id + << ", stream " << header->stream_id; + continue; + } + + stream_it = client_session->audio_streams.emplace(header->stream_id, std::move(client_mic_output)).first; + BOOST_LOG(info) << "Created audio stream " << header->stream_id << " for client " << header->client_id; + } + + // Decode and output audio + std::vector packet_decode_buffer(960); // 20ms at 48kHz mono + int decoded_samples = opus_decode_float( + reinterpret_cast(client_session->opus_decoder.get()), + opus_data, opus_size, packet_decode_buffer.data(), packet_decode_buffer.size(), 0 + ); + + if (decoded_samples > 0) { + packet_decode_buffer.resize(decoded_samples); + stream_it->second->output_samples(packet_decode_buffer); + + // Also output to virtual microphone for lobby chat + if (virtual_mic_available) { + virtual_mic_output->output_samples(packet_decode_buffer); + } + } else { + BOOST_LOG(warning) << "Failed to decode Opus data from client " << header->client_id + << ", stream " << header->stream_id << ": " << opus_strerror(decoded_samples); + } + } + + // Cleanup all client sessions + { + std::lock_guard lock(mic_clients_mutex); + for (const auto& [client_id, client_session] : mic_clients) { + for (auto& [stream_id, cleanup_mic_output] : client_session->audio_streams) { + cleanup_mic_output->stop(); + } + if (client_session->opus_decoder) { + opus_decoder_destroy(reinterpret_cast(client_session->opus_decoder.release())); + } + } + mic_clients.clear(); + } + + // Stop virtual microphone output + if (virtual_mic_available) { + virtual_mic_output->stop(); + } + + BOOST_LOG(info) << "Stopped microphone receiver thread"sv; + } } // namespace audio diff --git a/src/audio.h b/src/audio.h index 2afb42e5f70..a43f8a33bb3 100644 --- a/src/audio.h +++ b/src/audio.h @@ -72,6 +72,7 @@ namespace audio { using audio_ctx_ref_t = safe::shared_t::ptr_t; void capture(safe::mail_t mail, config_t config, void *channel_data); + void mic_receive(safe::mail_t mail, config_t config, void *channel_data); /** * @brief Get the reference to the audio context. diff --git a/src/config.cpp b/src/config.cpp index c1dba388ef1..4c3c8665bec 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -510,8 +510,18 @@ namespace config { audio_t audio { {}, // audio_sink {}, // virtual_sink + {}, // mic_sink true, // stream audio true, // install_steam_drivers + false, // enable_mic_passthrough + + // Advanced microphone settings + 8, // mic_max_clients + 2, // mic_max_streams_per_client + false, // mic_encryption_enabled + "OPUS,PCM", // mic_supported_codecs + "16000,48000", // mic_supported_sample_rates + 64000, // mic_default_bitrate }; stream_t stream { @@ -1162,8 +1172,10 @@ namespace config { string_f(vars, "audio_sink", audio.sink); string_f(vars, "virtual_sink", audio.virtual_sink); + string_f(vars, "mic_sink", audio.mic_sink); bool_f(vars, "stream_audio", audio.stream); bool_f(vars, "install_steam_audio_drivers", audio.install_steam_drivers); + bool_f(vars, "enable_mic_passthrough", audio.enable_mic_passthrough); string_restricted_f(vars, "origin_web_ui_allowed", nvhttp.origin_web_ui_allowed, {"pc"sv, "lan"sv, "wan"sv}); diff --git a/src/config.h b/src/config.h index 2e088ea757e..02bca1625d8 100644 --- a/src/config.h +++ b/src/config.h @@ -146,8 +146,18 @@ namespace config { struct audio_t { std::string sink; std::string virtual_sink; + std::string mic_sink; bool stream; bool install_steam_drivers; + bool enable_mic_passthrough; + + // Advanced microphone settings for multi-client support + int mic_max_clients; + int mic_max_streams_per_client; + bool mic_encryption_enabled; + std::string mic_supported_codecs; + std::string mic_supported_sample_rates; + int mic_default_bitrate; }; constexpr int ENCRYPTION_MODE_NEVER = 0; // Never use video encryption, even if the client supports it diff --git a/src/globals.h b/src/globals.h index 1617b7c6f51..58d41283809 100644 --- a/src/globals.h +++ b/src/globals.h @@ -47,6 +47,7 @@ namespace mail { MAIL(broadcast_shutdown); MAIL(video_packets); MAIL(audio_packets); + MAIL(mic_packets); MAIL(switch_display); // Local mail diff --git a/src/platform/common.h b/src/platform/common.h index 28704bb128e..5fcf244b746 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -550,12 +550,23 @@ namespace platf { virtual ~mic_t() = default; }; + class mic_output_t { + public: + virtual int output_samples(const std::vector &frame_buffer) = 0; + virtual int start() = 0; + virtual int stop() = 0; + + virtual ~mic_output_t() = default; + }; + class audio_control_t { public: virtual int set_sink(const std::string &sink) = 0; virtual std::unique_ptr microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size) = 0; + virtual std::unique_ptr mic_output(int channels, std::uint32_t sample_rate, const std::string &device_name) = 0; + /** * @brief Check if the audio sink is available in the system. * @param sink Sink to be checked. @@ -565,6 +576,9 @@ namespace platf { virtual std::optional sink_info() = 0; + virtual int create_virtual_microphone(const std::string &virtual_mic_name = "sunshine-virtual-mic") = 0; + virtual int setup_virtual_mic_loopback(const std::string &virtual_mic_name = "sunshine-virtual-mic") = 0; + virtual ~audio_control_t() = default; }; diff --git a/src/platform/linux/audio.cpp b/src/platform/linux/audio.cpp index 0e53e939b2a..8e8871712cc 100644 --- a/src/platform/linux/audio.cpp +++ b/src/platform/linux/audio.cpp @@ -68,6 +68,72 @@ namespace platf { } }; + struct mic_output_pa_t: public mic_output_t { + util::safe_ptr output; + std::string device_name; + bool started = false; + + mic_output_pa_t(int channels, std::uint32_t sample_rate, const std::string &dev_name) + : device_name(dev_name) { + + pa_sample_spec ss {PA_SAMPLE_FLOAT32, sample_rate, (std::uint8_t)channels}; + pa_channel_map pa_map; + + pa_map.channels = channels; + for (int i = 0; i < channels; i++) { + pa_map.map[i] = PA_CHANNEL_POSITION_MONO; + } + + pa_buffer_attr pa_attr = { + .maxlength = uint32_t(-1), + .tlength = uint32_t(-1), + .prebuf = uint32_t(-1), + .minreq = uint32_t(-1), + .fragsize = uint32_t(-1) + }; + + int status; + output.reset( + pa_simple_new(nullptr, "sunshine-mic", PA_STREAM_PLAYBACK, + device_name.c_str(), "sunshine-mic-output", + &ss, &pa_map, &pa_attr, &status) + ); + + if (!output) { + BOOST_LOG(error) << "Failed to create PulseAudio mic output: "sv << pa_strerror(status); + } + } + + int output_samples(const std::vector &frame_buffer) override { + if (!output || !started) { + return -1; + } + + int status; + if (pa_simple_write(output.get(), frame_buffer.data(), + frame_buffer.size() * sizeof(float), &status)) { + BOOST_LOG(error) << "pa_simple_write() failed: "sv << pa_strerror(status); + return -1; + } + + return 0; + } + + int start() override { + started = output != nullptr; + return started ? 0 : -1; + } + + int stop() override { + started = false; + if (output) { + int status; + pa_simple_drain(output.get(), &status); + } + return 0; + } + }; + std::unique_ptr microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size, std::string source_name) { auto mic = std::make_unique(); @@ -193,6 +259,11 @@ namespace platf { std::uint32_t surround71 = PA_INVALID_INDEX; } index; + struct { + std::uint32_t virtual_mic = PA_INVALID_INDEX; + std::uint32_t virtual_mic_loopback = PA_INVALID_INDEX; + } virtual_mic_index; + std::unique_ptr> events; std::unique_ptr> events_cb; @@ -459,6 +530,92 @@ namespace platf { return ::platf::microphone(mapping, channels, sample_rate, frame_size, get_monitor_name(sink_name)); } + std::unique_ptr mic_output(int channels, std::uint32_t sample_rate, const std::string &device_name) override { + return std::make_unique(channels, sample_rate, device_name); + } + + int create_virtual_microphone(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + if (virtual_mic_index.virtual_mic != PA_INVALID_INDEX) { + BOOST_LOG(info) << "Virtual microphone already exists"; + return virtual_mic_index.virtual_mic; + } + + auto alarm = safe::make_alarm(); + + BOOST_LOG(info) << "Creating virtual microphone: " << virtual_mic_name; + + std::string module_args = "sink_name=" + virtual_mic_name + + " sink_properties=device.description=\"Sunshine Virtual Microphone\"" + + " rate=48000 channels=1 format=float32le"; + + op_t op { + pa_context_load_module( + ctx.get(), + "module-null-sink", + module_args.c_str(), + cb_i, + alarm.get() + ) + }; + + if (!op) { + BOOST_LOG(error) << "Failed to create virtual microphone module operation"; + return -1; + } + + alarm->wait(); + auto module_index = *alarm->status(); + if (module_index == PA_INVALID_INDEX) { + BOOST_LOG(error) << "Failed to load virtual microphone module"; + return -1; + } + + virtual_mic_index.virtual_mic = module_index; + BOOST_LOG(info) << "Virtual microphone created with module index: " << module_index; + return module_index; + } + + int setup_virtual_mic_loopback(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + if (virtual_mic_index.virtual_mic_loopback != PA_INVALID_INDEX) { + BOOST_LOG(info) << "Virtual microphone loopback already exists"; + return virtual_mic_index.virtual_mic_loopback; + } + + auto alarm = safe::make_alarm(); + + BOOST_LOG(info) << "Setting up virtual microphone loopback"; + + std::string module_args = "source=" + virtual_mic_name + ".monitor" + + " sink=@DEFAULT_SINK@" + + " latency_msec=1"; + + op_t op { + pa_context_load_module( + ctx.get(), + "module-loopback", + module_args.c_str(), + cb_i, + alarm.get() + ) + }; + + if (!op) { + BOOST_LOG(error) << "Failed to create loopback module operation"; + return -1; + } + + alarm->wait(); + auto module_index = *alarm->status(); + if (module_index == PA_INVALID_INDEX) { + BOOST_LOG(error) << "Failed to load loopback module"; + return -1; + } + + virtual_mic_index.virtual_mic_loopback = module_index; + BOOST_LOG(info) << "Virtual microphone loopback created with module index: " << module_index; + return module_index; + } + bool is_sink_available(const std::string &sink) override { BOOST_LOG(warning) << "audio_control_t::is_sink_available() unimplemented: "sv << sink; return true; @@ -499,6 +656,9 @@ namespace platf { unload_null(index.surround51); unload_null(index.surround71); + unload_null(virtual_mic_index.virtual_mic); + unload_null(virtual_mic_index.virtual_mic_loopback); + if (worker.joinable()) { pa_context_disconnect(ctx.get()); diff --git a/src/platform/macos/microphone.mm b/src/platform/macos/microphone.mm index 06b9c19a899..8c265785a52 100644 --- a/src/platform/macos/microphone.mm +++ b/src/platform/macos/microphone.mm @@ -40,6 +40,62 @@ capture_e sample(std::vector &sample_in) override { } }; + struct av_mic_output_t: public mic_output_t { + AVAudio *av_audio_output; + std::string device_name; + bool started = false; + + av_mic_output_t(int channels, std::uint32_t sample_rate, const std::string &dev_name) + : device_name(dev_name) { + + av_audio_output = [[AVAudio alloc] init]; + + AVCaptureDevice *output_device = nullptr; + if (!device_name.empty() && device_name != "default") { + output_device = [AVAudio findMicrophone:[NSString stringWithUTF8String:device_name.c_str()]]; + } + + if (!output_device) { + output_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + } + + if ([av_audio_output setupMicrophone:output_device sampleRate:sample_rate frameSize:960 channels:channels]) { + BOOST_LOG(error) << "Failed to setup microphone output device."sv; + [av_audio_output release]; + av_audio_output = nullptr; + } + } + + int output_samples(const std::vector &frame_buffer) override { + if (!av_audio_output || !started) { + return -1; + } + + // For macOS, we would need to implement audio output through Core Audio + // This is a simplified placeholder - real implementation would need + // Audio Queue Services or Audio Unit for output + BOOST_LOG(debug) << "Outputting " << frame_buffer.size() << " audio samples"sv; + return 0; + } + + int start() override { + started = av_audio_output != nullptr; + return started ? 0 : -1; + } + + int stop() override { + started = false; + return 0; + } + + ~av_mic_output_t() override { + stop(); + if (av_audio_output) { + [av_audio_output release]; + } + } + }; + struct macos_audio_control_t: public audio_control_t { AVCaptureDevice *audio_capture_device {}; @@ -78,6 +134,10 @@ int set_sink(const std::string &sink) override { return mic; } + std::unique_ptr mic_output(int channels, std::uint32_t sample_rate, const std::string &device_name) override { + return std::make_unique(channels, sample_rate, device_name); + } + bool is_sink_available(const std::string &sink) override { BOOST_LOG(warning) << "audio_control_t::is_sink_available() unimplemented: "sv << sink; return true; @@ -88,6 +148,16 @@ bool is_sink_available(const std::string &sink) override { return sink; } + + int create_virtual_microphone(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + BOOST_LOG(warning) << "Virtual microphone creation not implemented on macOS yet"; + return -1; + } + + int setup_virtual_mic_loopback(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + BOOST_LOG(warning) << "Virtual microphone loopback not implemented on macOS yet"; + return -1; + } }; std::unique_ptr audio_control() { diff --git a/src/platform/windows/audio.cpp b/src/platform/windows/audio.cpp index 20e600acaa9..69e16605fcb 100644 --- a/src/platform/windows/audio.cpp +++ b/src/platform/windows/audio.cpp @@ -680,6 +680,193 @@ namespace platf::audio { HANDLE mmcss_task_handle = NULL; }; + class mic_output_wasapi_t: public mic_output_t { + public: + com_ptr_t device_enum; + com_ptr_t device; + com_ptr_t audio_client; + com_ptr_t render_client; + WAVEFORMATEX *wave_format = nullptr; + HANDLE event_handle = NULL; + std::string device_name; + bool started = false; + + mic_output_wasapi_t(int channels, std::uint32_t sample_rate, const std::string &dev_name) + : device_name(dev_name) { + + auto hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), + nullptr, + CLSCTX_ALL, + IID_PPV_ARGS(&device_enum) + ); + + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to create device enumerator: "sv << std::hex << hr; + return; + } + + // Use default device if device_name is empty or "default" + if (device_name.empty() || device_name == "default") { + hr = device_enum->GetDefaultAudioEndpoint(eRender, eConsole, &device); + } else { + // Try to find device by name + com_ptr_t collection; + hr = device_enum->EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE, &collection); + if (SUCCEEDED(hr)) { + UINT count; + collection->GetCount(&count); + for (UINT i = 0; i < count && !device; i++) { + com_ptr_t temp_device; + collection->Item(i, &temp_device); + + com_ptr_t prop_store; + temp_device->OpenPropertyStore(STGM_READ, &prop_store); + + PROPVARIANT prop_var; + PropVariantInit(&prop_var); + prop_store->GetValue(PKEY_Device_FriendlyName, &prop_var); + + std::wstring name = prop_var.pwszVal; + std::string device_friendly_name(name.begin(), name.end()); + + if (device_friendly_name.find(device_name) != std::string::npos) { + device = temp_device; + } + + PropVariantClear(&prop_var); + } + } + + if (!device) { + hr = device_enum->GetDefaultAudioEndpoint(eRender, eConsole, &device); + } + } + + if (FAILED(hr) || !device) { + BOOST_LOG(error) << "Failed to get audio device: "sv << std::hex << hr; + return; + } + + hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, (void**)&audio_client); + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to activate audio client: "sv << std::hex << hr; + return; + } + + // Set up wave format for mono float at specified sample rate + WAVEFORMATEXTENSIBLE wave_ex = {}; + wave_ex.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + wave_ex.Format.nChannels = channels; + wave_ex.Format.nSamplesPerSec = sample_rate; + wave_ex.Format.wBitsPerSample = 32; + wave_ex.Format.nBlockAlign = channels * sizeof(float); + wave_ex.Format.nAvgBytesPerSec = sample_rate * wave_ex.Format.nBlockAlign; + wave_ex.Format.cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX); + wave_ex.Samples.wValidBitsPerSample = 32; + wave_ex.dwChannelMask = SPEAKER_FRONT_CENTER; + wave_ex.SubFormat = KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; + + wave_format = (WAVEFORMATEX*)&wave_ex; + + event_handle = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (!event_handle) { + BOOST_LOG(error) << "Failed to create event handle"; + return; + } + + hr = audio_client->Initialize( + AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_EVENTCALLBACK, + 200000, // 20ms buffer + 0, + wave_format, + nullptr + ); + + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to initialize audio client: "sv << std::hex << hr; + return; + } + + hr = audio_client->SetEventHandle(event_handle); + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to set event handle: "sv << std::hex << hr; + return; + } + + hr = audio_client->GetService(IID_PPV_ARGS(&render_client)); + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to get render client: "sv << std::hex << hr; + return; + } + } + + int output_samples(const std::vector &frame_buffer) override { + if (!audio_client || !render_client || !started) { + return -1; + } + + UINT32 buffer_frames; + audio_client->GetBufferSize(&buffer_frames); + + UINT32 padding; + audio_client->GetCurrentPadding(&padding); + + UINT32 available_frames = buffer_frames - padding; + UINT32 frames_to_write = std::min(available_frames, (UINT32)frame_buffer.size()); + + if (frames_to_write == 0) { + return 0; + } + + BYTE *buffer_ptr; + auto hr = render_client->GetBuffer(frames_to_write, &buffer_ptr); + if (FAILED(hr)) { + return -1; + } + + memcpy(buffer_ptr, frame_buffer.data(), frames_to_write * sizeof(float)); + + hr = render_client->ReleaseBuffer(frames_to_write, 0); + if (FAILED(hr)) { + return -1; + } + + return 0; + } + + int start() override { + if (!audio_client) { + return -1; + } + + auto hr = audio_client->Start(); + if (FAILED(hr)) { + BOOST_LOG(error) << "Failed to start audio client: "sv << std::hex << hr; + return -1; + } + + started = true; + return 0; + } + + int stop() override { + if (audio_client && started) { + audio_client->Stop(); + started = false; + } + return 0; + } + + ~mic_output_wasapi_t() { + stop(); + if (event_handle) { + CloseHandle(event_handle); + } + } + }; + class audio_control_t: public ::platf::audio_control_t { public: std::optional sink_info() override { @@ -778,6 +965,10 @@ namespace platf::audio { return mic; } + std::unique_ptr mic_output(int channels, std::uint32_t sample_rate, const std::string &device_name) override { + return std::make_unique(channels, sample_rate, device_name); + } + /** * If the requested sink is a virtual sink, meaning no speakers attached to * the host, then we can seamlessly set the format to stereo and surround sound. @@ -1142,6 +1333,16 @@ namespace platf::audio { ~audio_control_t() override { } + int create_virtual_microphone(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + BOOST_LOG(warning) << "Virtual microphone creation not implemented on Windows yet"; + return -1; + } + + int setup_virtual_mic_loopback(const std::string &virtual_mic_name = "sunshine-virtual-mic") override { + BOOST_LOG(warning) << "Virtual microphone loopback not implemented on Windows yet"; + return -1; + } + policy_t policy; audio::device_enum_t device_enum; std::string assigned_sink; diff --git a/src/rtsp.cpp b/src/rtsp.cpp index cd43dd0c5c2..d460d3f1007 100644 --- a/src/rtsp.cpp +++ b/src/rtsp.cpp @@ -755,6 +755,19 @@ namespace rtsp_stream { // Tell the client about our supported features ss << "a=x-ss-general.featureFlags:" << (uint32_t) platf::get_capabilities() << std::endl; + + // Advertise microphone support capabilities if enabled + if (config::audio.enable_mic_passthrough) { + ss << "a=x-ss-mic.version:1" << std::endl; + ss << "a=x-ss-mic.enabled:1" << std::endl; + ss << "a=x-ss-mic.codecs:" << config::audio.mic_supported_codecs << std::endl; + ss << "a=x-ss-mic.sampleRates:" << config::audio.mic_supported_sample_rates << std::endl; + ss << "a=x-ss-mic.channels:1,2" << std::endl; + ss << "a=x-ss-mic.defaultBitrate:" << config::audio.mic_default_bitrate << std::endl; + ss << "a=x-ss-mic.maxClients:" << config::audio.mic_max_clients << std::endl; + ss << "a=x-ss-mic.maxStreamsPerClient:" << config::audio.mic_max_streams_per_client << std::endl; + ss << "a=x-ss-mic.encryption:" << (config::audio.mic_encryption_enabled ? "1" : "0") << std::endl; + } // Always request new control stream encryption if the client supports it uint32_t encryption_flags_supported = SS_ENC_CONTROL_V2 | SS_ENC_AUDIO; @@ -850,6 +863,8 @@ namespace rtsp_stream { port = net::map_port(stream::VIDEO_STREAM_PORT); } else if (type == "control"sv) { port = net::map_port(stream::CONTROL_PORT); + } else if (type == "microphone"sv) { + port = net::map_port(stream::MIC_STREAM_PORT); } else { cmd_not_found(sock, session, std::move(req)); @@ -956,6 +971,9 @@ namespace rtsp_stream { std::int64_t configuredBitrateKbps; config.audio.flags[audio::config_t::HOST_AUDIO] = session.host_audio; + + // Enable client microphone if configured and supported + session.client_mic_enabled = config::audio.enable_mic_passthrough; try { config.audio.channels = util::from_view(args.at("x-nv-audio.surround.numChannels"sv)); config.audio.mask = util::from_view(args.at("x-nv-audio.surround.channelMask"sv)); diff --git a/src/rtsp.h b/src/rtsp.h index 2303b96b6b3..d5f21acfc3d 100644 --- a/src/rtsp.h +++ b/src/rtsp.h @@ -24,6 +24,7 @@ namespace rtsp_stream { uint32_t control_connect_data; bool host_audio; + bool client_mic_enabled; std::string unique_id; int width; int height; diff --git a/src/stream.cpp b/src/stream.cpp index e6bb5ebfde2..782427cae35 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -49,6 +49,8 @@ extern "C" { #define IDX_SET_MOTION_EVENT 13 #define IDX_SET_RGB_LED 14 #define IDX_SET_ADAPTIVE_TRIGGERS 15 +#define IDX_MIC_DATA 16 +#define IDX_MIC_CONFIG 17 static const short packetTypes[] = { 0x0305, // Start A @@ -67,6 +69,8 @@ static const short packetTypes[] = { 0x5501, // Set motion event (Sunshine protocol extension) 0x5502, // Set RGB LED (Sunshine protocol extension) 0x5503, // Set Adaptive triggers (Sunshine protocol extension) + 0x5504, // Microphone data (Sunshine protocol extension) + 0x5505, // Microphone config (Sunshine protocol extension) }; namespace asio = boost::asio; @@ -81,7 +85,8 @@ namespace stream { enum class socket_e : int { video, ///< Video - audio ///< Audio + audio, ///< Audio + microphone ///< Microphone }; #pragma pack(push, 1) @@ -331,11 +336,13 @@ namespace stream { std::thread video_thread; std::thread audio_thread; std::thread control_thread; + std::thread mic_thread; asio::io_context io_context; udp::socket video_sock {io_context}; udp::socket audio_sock {io_context}; + udp::socket mic_sock {io_context}; control_server_t control_server; }; @@ -349,6 +356,7 @@ namespace stream { std::thread audioThread; std::thread videoThread; + std::thread micThread; std::chrono::steady_clock::time_point pingTimeout; @@ -1584,6 +1592,43 @@ namespace stream { shutdown_event->raise(true); } + void micReceiveThread(udp::socket &sock) { + auto shutdown_event = mail::man->event(mail::broadcast_shutdown); + auto mic_packets = mail::man->queue(mail::mic_packets); + + BOOST_LOG(info) << "Starting microphone receive thread"sv; + + while (!shutdown_event->peek()) { + std::array buffer; + udp::endpoint sender_endpoint; + boost::system::error_code ec; + + auto bytes_received = sock.receive_from(asio::buffer(buffer), sender_endpoint, 0, ec); + + if (ec) { + if (ec == boost::asio::error::operation_aborted) { + break; + } + BOOST_LOG(warning) << "Microphone receive error: " << ec.message(); + continue; + } + + if (bytes_received < 8) { + continue; // Too small to be a valid packet + } + + // Extract microphone data and forward to processing + auto mic_packet = std::make_pair( + (void*)buffer.data(), + audio::buffer_t(buffer.begin(), buffer.begin() + bytes_received) + ); + + mic_packets->raise(std::move(mic_packet)); + } + + BOOST_LOG(info) << "Microphone receive thread ended"sv; + } + void audioBroadcastThread(udp::socket &sock) { auto shutdown_event = mail::man->event(mail::broadcast_shutdown); auto packets = mail::man->queue(mail::audio_packets); @@ -1694,6 +1739,7 @@ namespace stream { auto control_port = net::map_port(CONTROL_PORT); auto video_port = net::map_port(VIDEO_STREAM_PORT); auto audio_port = net::map_port(AUDIO_STREAM_PORT); + auto mic_port = net::map_port(MIC_STREAM_PORT); if (ctx.control_server.bind(address_family, control_port)) { BOOST_LOG(error) << "Couldn't bind Control server to port ["sv << control_port << "], likely another process already bound to the port"sv; @@ -1737,12 +1783,30 @@ namespace stream { return -1; } + if (config::audio.enable_mic_passthrough) { + ctx.mic_sock.open(protocol, ec); + if (ec) { + BOOST_LOG(fatal) << "Couldn't open socket for Microphone server: "sv << ec.message(); + return -1; + } + + ctx.mic_sock.bind(udp::endpoint(protocol, mic_port), ec); + if (ec) { + BOOST_LOG(fatal) << "Couldn't bind Microphone server to port ["sv << mic_port << "]: "sv << ec.message(); + return -1; + } + } + ctx.message_queue_queue = std::make_shared(30); ctx.video_thread = std::thread {videoBroadcastThread, std::ref(ctx.video_sock)}; ctx.audio_thread = std::thread {audioBroadcastThread, std::ref(ctx.audio_sock)}; ctx.control_thread = std::thread {controlBroadcastThread, &ctx.control_server}; + if (config::audio.enable_mic_passthrough) { + ctx.mic_thread = std::thread {micReceiveThread, std::ref(ctx.mic_sock)}; + } + ctx.recv_thread = std::thread {recvThread, std::ref(ctx)}; return 0; @@ -1765,6 +1829,9 @@ namespace stream { ctx.video_sock.close(); ctx.audio_sock.close(); + if (ctx.mic_sock.is_open()) { + ctx.mic_sock.close(); + } video_packets.reset(); audio_packets.reset(); @@ -1777,6 +1844,12 @@ namespace stream { ctx.audio_thread.join(); BOOST_LOG(debug) << "Waiting for main control thread to end..."sv; ctx.control_thread.join(); + + if (ctx.mic_thread.joinable()) { + BOOST_LOG(debug) << "Waiting for microphone thread to end..."sv; + ctx.mic_thread.join(); + } + BOOST_LOG(debug) << "All broadcasting threads ended"sv; broadcast_shutdown_event->reset(); @@ -1914,6 +1987,12 @@ namespace stream { session.videoThread.join(); BOOST_LOG(debug) << "Waiting for audio to end..."sv; session.audioThread.join(); + + if (session.micThread.joinable()) { + BOOST_LOG(debug) << "Waiting for microphone to end..."sv; + session.micThread.join(); + } + BOOST_LOG(debug) << "Waiting for control to end..."sv; session.controlEnd.view(); // Reset input on session stop to avoid stuck repeated keys @@ -1970,6 +2049,10 @@ namespace stream { session.audioThread = std::thread {audioThread, &session}; session.videoThread = std::thread {videoThread, &session}; + + if (config::audio.enable_mic_passthrough) { + session.micThread = std::thread {audio::mic_receive, session.mail, session.config.audio, nullptr}; + } session.state.store(state_e::RUNNING, std::memory_order_relaxed); diff --git a/src/stream.h b/src/stream.h index 53afff4fabe..831126ba4ac 100644 --- a/src/stream.h +++ b/src/stream.h @@ -19,6 +19,27 @@ namespace stream { constexpr auto VIDEO_STREAM_PORT = 9; constexpr auto CONTROL_PORT = 10; constexpr auto AUDIO_STREAM_PORT = 11; + constexpr auto MIC_STREAM_PORT = 13; + + // Microphone protocol constants + constexpr uint16_t MIC_PROTOCOL_VERSION = 0x0001; + constexpr uint16_t MIC_PACKET_AUDIO = 0x0001; + constexpr uint16_t MIC_PACKET_CONTROL = 0x0002; + constexpr uint16_t MIC_FLAG_ENCRYPTED = 0x0001; + constexpr uint16_t MIC_FLAG_FEC = 0x0002; + + // Microphone packet header for client identification and multi-stream support + struct mic_packet_header_t { + uint16_t version; // Protocol version (0x0001) + uint16_t packet_type; // 0x0001 = audio data, 0x0002 = control + uint32_t client_id; // Client session identifier + uint16_t stream_id; // Audio stream identifier (for multiple mics) + uint16_t sequence; // Packet sequence number + uint32_t timestamp; // Audio timestamp + uint16_t payload_size; // Size of audio payload after header + uint16_t flags; // Optional flags (encryption, FEC, etc.) + // Audio payload follows + }; struct session_t; diff --git a/src/upnp.cpp b/src/upnp.cpp index 625a7a69eba..b9a55db55b4 100644 --- a/src/upnp.cpp +++ b/src/upnp.cpp @@ -72,12 +72,14 @@ namespace upnp { auto gs_http = std::to_string(net::map_port(nvhttp::PORT_HTTP)); auto gs_https = std::to_string(net::map_port(nvhttp::PORT_HTTPS)); auto wm_http = std::to_string(net::map_port(confighttp::PORT_HTTPS)); + auto mic = std::to_string(net::map_port(stream::MIC_STREAM_PORT)); mappings.assign({ {{rtsp, rtsp, "TCP"s}, "Sunshine - RTSP"s}, {{video, video, "UDP"s}, "Sunshine - Video"s}, {{audio, audio, "UDP"s}, "Sunshine - Audio"s}, {{control, control, "UDP"s}, "Sunshine - Control"s}, + {{mic, mic, "UDP"s}, "Sunshine - Microphone"s}, {{gs_http, gs_http, "TCP"s}, "Sunshine - Client HTTP"s}, {{gs_https, gs_https, "TCP"s}, "Sunshine - Client HTTPS"s}, }); diff --git a/src_assets/common/assets/web/configs/tabs/Network.vue b/src_assets/common/assets/web/configs/tabs/Network.vue index 4fac049dbbd..d1eaae94fdb 100644 --- a/src_assets/common/assets/web/configs/tabs/Network.vue +++ b/src_assets/common/assets/web/configs/tabs/Network.vue @@ -88,7 +88,7 @@ const effectivePort = computed(() => +config.value?.port ?? defaultMoonlightPort {{ $t('config.port_udp') }} - {{+effectivePort + 9}} - {{+effectivePort + 11}} + {{+effectivePort + 9}} - {{+effectivePort + 13}}