First implementation of the visualization output plugin.

This commit contains the first implementation of an output
plugin that streams sound analysis to clients (presumably
visualizers of some sort).

.github/workflows/build.yml

@@ -88,7 +88,8 @@ jobs:
             libupnp-dev \
             libsqlite3-dev \
             libchromaprint-dev \
-            libgcrypt20-dev
+            libgcrypt20-dev \
+            libfftw3-dev
 
       - name: Full Build
         uses: BSFishy/[email protected]
@@ -152,7 +153,8 @@ jobs:
             libvorbis \
             faad2 \
             wavpack \
-            libmpdclient
+            libmpdclient \
+            fftw
 
       - name: Build
         uses: BSFishy/[email protected]
@@ -193,6 +195,7 @@ jobs:
             dbus:p
             faad2:p
             ffmpeg:p
+            fftw:p
             fmt:p
             flac:p
             gtest:p

.github/workflows/build_android.yml

@@ -40,7 +40,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends \
             ninja-build \
-            quilt 
+            quilt
           pip3 install --user meson==1.3.0
             
       - name: Build
@@ -49,7 +49,7 @@ jobs:
           cd ./output/android
           ../../android/build.py $ANDROID_SDK_ROOT $ANDROID_NDK_LATEST_HOME arm64-v8a \
             --buildtype=debugoptimized -Db_ndebug=true \
-            -Dwrap_mode=forcefallback 
+            -Dwrap_mode=forcefallback -Dvisualization=false
 
           cd -
           cd ./android

meson_options.txt

@@ -186,6 +186,8 @@ option('shout', type: 'feature', description: 'Shoutcast streaming support using
 option('snapcast', type: 'boolean', value: true, description: 'Snapcast output plugin')
 option('sndio', type: 'feature', description: 'sndio output plugin')
 option('solaris_output', type: 'feature', description: 'Solaris /dev/audio support')
+option('visualization', type: 'boolean', value: true, description: 'Visualization output plugin')
+option('fftw3', type: 'feature', description: 'FFTW support')
 
 #
 # Misc libraries

src/lib/fmt/ThreadIdFormatter.hxx

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// author: Max Kellermann <[email protected]>
+
+#ifndef THREAD_ID_FORMATTER_HXX
+#define THREAD_ID_FORMATTER_HXX
+
+#include <fmt/format.h>
+#include <sstream>
+#include <thread>
+
+template<>
+struct fmt::formatter<std::thread::id> : formatter<string_view>
+{
+	template<typename FormatContext>
+	auto format(std::thread::id id, FormatContext &ctx) {
+		std::stringstream stm;
+		stm << id;
+		return formatter<string_view>::format(stm.str(), ctx);
+	}
+};
+
+#endif // THREAD_ID_FORMATTER_HXX

src/output/Registry.cxx

@@ -23,6 +23,7 @@
 #include "plugins/ShoutOutputPlugin.hxx"
 #include "plugins/sles/SlesOutputPlugin.hxx"
 #include "plugins/SolarisOutputPlugin.hxx"
+#include "plugins/visualization/VisualizationOutputPlugin.hxx"
 #ifdef ENABLE_WINMM_OUTPUT
 #include "plugins/WinmmOutputPlugin.hxx"
 #endif
@@ -89,6 +90,9 @@ constinit const AudioOutputPlugin *const audio_output_plugins[] = {
 #endif
 #ifdef ENABLE_WASAPI_OUTPUT
 	&wasapi_output_plugin,
+#endif
+#ifdef ENABLE_VISUALIZATION_OUTPUT
+	&visualization_output_plugin,
 #endif
 	nullptr
 };

src/output/plugins/meson.build

@@ -160,6 +160,24 @@ else
   wasapi_dep = dependency('', required: false)
 endif
 
+libfftw3_dep = dependency('fftw3f', version: '>= 3.3.8', required: get_option('fftw3'))
+output_features.set('ENABLE_FFTW3', libfftw3_dep.found())
+output_features.set('ENABLE_VISUALIZATION_OUTPUT', get_option('visualization'))
+if get_option('visualization')
+  if not libfftw3_dep.found()
+    error('libfftw3 not available, but is required for the visualization plugin')
+  endif
+  output_plugins_sources += [
+    'visualization/VisualizationOutputPlugin.cxx',
+    'visualization/SoundAnalysis.cxx',
+    'visualization/SoundInfoCache.cxx',
+    'visualization/VisualizationServer.cxx',
+    'visualization/VisualizationClient.cxx',
+    'visualization/Protocol.cxx',
+  ]
+  output_plugins_deps += [ event_dep, net_dep, libfftw3_dep ]
+endif
+
 output_plugins = static_library(
   'output_plugins',
   output_plugins_sources,

src/output/plugins/visualization/LowLevelProtocol.hxx

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright The Music Player Daemon Project
+
+#ifndef LOW_LEVEL_PROTOCOL_HXX_INCLUDED
+#define LOW_LEVEL_PROTOCOL_HXX_INCLUDED
+
+#include "util/PackedBigEndian.hxx"
+
+#include <fftw3.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+
+namespace Visualization {
+
+/*  Write a uint16_t to an output iterator over byte in wire format; return the
+ *  iterator in its new position
+ */
+template <typename OutIter>
+OutIter
+SerializeU16(uint16_t n, OutIter pout) {
+	auto m = PackedBE16(n);
+	auto p = (std::byte*)(&m);
+	return std::copy(p, p + 2, pout);
+}
+
+static_assert(std::numeric_limits<float>::is_iec559);
+
+/* Convert an IEEE 754 single-precision floating-point number to wire format;
+ * write it to an output iterator & return the iterator in its new position
+ */
+template <typename OutIter>
+OutIter
+SerializeFloat(float f, OutIter pout) {
+	auto m = PackedBE32(*(uint32_t*)&f);
+	auto p = (std::byte*)(&m);
+	return std::copy(p, p + 4, pout);
+}
+
+/* Convert an fftwf_complex to wire format; write it to an output iterator &
+ * return the iterator in its new position
+ */
+template <typename OutIter>
+OutIter
+SerializeComplex(const fftwf_complex c, OutIter pout) {
+	auto r = PackedBE32(*(const uint32_t*)&(c[0]));
+	auto i = PackedBE32(*(const uint32_t*)&(c[1]));
+	auto pr = (std::byte*)(&r);
+	auto pi = (std::byte*)(&i);
+	pout = std::copy(pr, pr + 4, pout);
+	return std::copy(pi, pi + 4, pout);
+}
+
+} // namespace Visualization
+
+#endif // LOW_LEVEL_PROTOCOL_HXX_INCLUDED

src/output/plugins/visualization/Protocol.cxx

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright The Music Player Daemon Project
+
+#include "Protocol.hxx"
+
+#include "Log.hxx"
+#include "util/ByteOrder.hxx"
+#include "util/Domain.hxx"
+
+Visualization::ParseResult
+Visualization::ParseClihlo(void *data,
+			   size_t length,
+			   ClientHello &clihlo) noexcept {
+	// CLIHLO payload is 6 bytes, header & footer are five more.
+	if (length < sizeof(ClientHello) + 5) {
+		return ParseResult::NEED_MORE_DATA;
+	}
+
+	uint8_t *buf = (uint8_t *)data;
+
+	uint16_t msg_type = FromBE16(*(uint16_t *)buf);
+	if (msg_type != 0) {
+		return ParseResult::ERROR;
+	}
+
+	buf += 2;
+	uint16_t payload_len = FromBE16(*(uint16_t *)buf);
+	if (payload_len != 6) {
+		return ParseResult::ERROR;
+	}
+
+	buf += 2;
+	clihlo.major_version = *buf++;
+	clihlo.minor_version = *buf++;
+
+	clihlo.requested_fps = FromBE16(*(uint16_t *)(buf));
+	buf += 2;
+	clihlo.tau = FromBE16(*(int16_t *)(buf));
+	buf += 2;
+
+	if (*buf != 0) {
+		return ParseResult::ERROR;
+	}
+
+	return ParseResult::OK;
+}

src/output/plugins/visualization/Protocol.hxx

@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright The Music Player Daemon Project
+
+#ifndef VISUALIZATION_PROTOCOL_HXX_INCLUDED
+#define VISUALIZATION_PROTOCOL_HXX_INCLUDED
+
+#include "LowLevelProtocol.hxx"
+#include "SoundAnalysis.hxx"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Visualization {
+
+/**
+ * \brief A parsed CLIHLO message
+ *
+ * \sa ParseCliHlo
+ *
+ *
+ * The visualization \ref vis_out_protocol "protocol" begins with the client
+ * connecting to the server & providing certain paramters of the sound analysis
+ * it would like to receive. That is done through the CLIHLO message (which see
+ * \a ref vis_out_protocol_proto_clihlo "here").
+ *
+ * See \a vis_out_protocol_timing "timing" for details on parameter tau.
+ *
+ *
+ */
+
+struct ClientHello {
+	/// Major protocol version the client would like to speak
+	uint8_t major_version;
+	/// Minor protocol version the client would like to speak
+	uint8_t minor_version;
+	/// The number of sound analyses per second the client would like to
+	/// receive (presumably the rate at which it is rendering frames, hence
+	/// the name "fps")
+	uint16_t requested_fps;
+	/// The desired offset (named "tau" in the documentation) between song
+	/// time and analysis time at each analysis performed
+	int16_t tau;
+};
+
+enum class ParseResult {
+	OK,
+	NEED_MORE_DATA,
+	ERROR,
+};
+
+/**
+ * \brief Attempt to parse a \ref vis_out_protocol_proto_clihlo "CLIHLO" message
+ * from the given buffer
+ *
+ * \param buf [in] An array of octets potentially containing the message
+ *
+ * \param length [in] The length of \a buf, in octets
+ *
+ * \param clihlo [out] A reference to a `client_hello_t` structure to be
+ * filled-in on successful execution
+ *
+ * \return ParseResult::OK if the message was successfully parsed,
+ * NEED_MORE_DATA if the message is incomplete, or ERROR if the message cannot
+ * be ready from \a buf
+ *
+ *
+ * CLIHLO is the first message in the protocol, sent by the client. See
+ * \ref vis_out_protocol_proto_clihlo "the protocol specification" for details,
+ * and \ref vis_out_protocol "Visualization Network Protocol" for discussion
+ * of the protocol altogether.
+ *
+ *
+ */
+
+ParseResult
+ParseClihlo(void *buf, size_t length, ClientHello &clihlo) noexcept;
+
+/// Serialize an SRVHLO message to wire format
+template <typename OutIter>
+void
+SerializeSrvhlo(std::byte major_ver, std::byte minor_ver, OutIter pout) {
+	using std::byte;
+
+	*pout++ = byte{0}; //
+	*pout++ = byte{1}; // message type
+	*pout++ = byte{0}; //
+	*pout++ = byte{2}; // payload length
+	*pout++ = major_ver;
+	*pout++ = minor_ver;
+	*pout++ = byte{0}; // check byte
+}
+
+/// Serialize a FRAME message header to wire format
+template <typename OutIter>
+OutIter
+SerializeSoundInfoFrameHeader(uint8_t num_chan,
+			      size_t num_samp,
+			      size_t num_freq,
+ OutIter pout) {
+
+	using std::byte;
+
+	// Start with the "magic number" allowing clients to "lock on" to the
+	// stream of sound info frames in the event of an error.
+	// See \ref vis_out_protocol_proto_msgs for details.
+	*pout++ = byte{0x63};
+	*pout++ = byte{0xac};
+	*pout++ = byte{0x84};
+	*pout++ = byte{0x03};
+
+	*pout++ = byte{16};
+	*pout++ = byte{0};
+
+	return SerializeU16(17 + 4 * num_chan * (num_samp + 3 * num_freq + 3),
+			    pout);
+}
+
+/// Serialize a FRAME message payload to wire format
+template <typename OutIter>
+void
+SerializeSoundInfoFrameFooter(OutIter pout) {
+	*pout = std::byte{0x00};
+}
+
+/// Serialize a FRAME message to wire format
+template <typename OutIter>
+void
+SerializeSoundInfoFrame(const Visualization::SoundAnalysis &a,
+			OutIter pout) {
+	pout = SerializeSoundInfoFrameHeader(a.NumChan(), a.NumSamp(),
+					     a.NumFreq(), pout);
+	pout = a.SerializeSoundInfoFramePayload(pout);
+	SerializeSoundInfoFrameFooter(pout);
+}
+
+} // namespace Visualization.
+
+#endif // VISUALIZATION_PROTOCOL_HXX_INCLUDED