Merge pull request #87 from jdibenes/mca5

1.0.27.0

README.md

@@ -17,7 +17,7 @@ HoloLens 2 server software and Python client library for streaming sensor data v
   - Gyroscope (deg/s)
   - Magnetometer
 - Front Camera (1920x1080 @ 30 FPS, RGB, H264 or HEVC encoded)
-- Microphone (2 channels @ 48000 Hz, 16-bit PCM, AAC encoded)
+- Microphone (2 channels @ 48000 Hz, 16-bit PCM, AAC encoded or 5 channels @ 48000 Hz, 32-bit Float)
 - Spatial Input (30 Hz)
   - Head Tracking
   - Eye Tracking
@@ -38,6 +38,7 @@ HoloLens 2 server software and Python client library for streaming sensor data v
 - Client can configure the focus, white balance, and exposure of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/viewer/client_ipc_rc.py).
 - Frame timestamps can be converted to [Windows FILETIME](https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-filetime) (UTC) for external synchronization. See [here](https://github.com/jdibenes/hl2ss/blob/main/viewer/client_ipc_rc.py).
 - Client can send messages to a Unity application using the plugin.
+- Server application can run in background (alongside other applications) when running in flat mode.
 - [C++ client library](https://github.com/jdibenes/hl2ss/tree/main/extensions).
 - [MATLAB client (MEX)](https://github.com/jdibenes/hl2ss/tree/main/extensions).
 
@@ -200,6 +201,7 @@ If you wish to create the server application appxbundle, right click the hl2ss p
 - Multiple streams can be active at the same time but only one client per stream is allowed.
 - Occasionally, the server might crash when accessing the Front Camera and RM Depth Long Throw streams simultaneously. See https://github.com/microsoft/HoloLens2ForCV/issues/142.
 - The RM Depth AHAT and RM Depth Long Throw streams cannot be accessed simultaneously.
+- Spatial Input is not supported in flat mode.
 
 ## References
 

etc/hl2_mca.txt

@@ -0,0 +1,12 @@
+# Microphone RAW (11 channels)
+#  0 ok (top left?)
+#  1 ok (top center?)
+#  2 ok (top right?)
+#  3 ok (bottom left?)
+#  4 ok (bottom right?)
+#  5 silent?
+#  6 silent?
+#  7 noise
+#  8 noise
+#  9 noise
+# 10 noise

extensions/client_cpp/hl2ss.cpp

@@ -1268,6 +1268,7 @@ uint8_t const SET_PV_EXPOSURE_PRIORITY_VIDEO  = 0x09;
 uint8_t const SET_PV_ISO_SPEED                = 0x0A;
 uint8_t const SET_PV_BACKLIGHT_COMPENSATION   = 0x0B;
 uint8_t const SET_PV_SCENE_MODE               = 0x0C;
+uint8_t const SET_FLAT_MODE                   = 0x0D;
 };
 
 ipc_rc::ipc_rc(char const* host, uint16_t port) : ipc(host, port)
@@ -1366,6 +1367,11 @@ void ipc_rc::set_pv_scene_mode(uint32_t mode)
     send(cmd_ipc_rc::SET_PV_SCENE_MODE, {mode});
 }
 
+void ipc_rc::set_flat_mode(uint32_t mode)
+{
+    send(cmd_ipc_rc::SET_FLAT_MODE, {mode});
+}
+
 //------------------------------------------------------------------------------
 // * Spatial Mapping
 //------------------------------------------------------------------------------

extensions/client_cpp/hl2ss.h

@@ -244,6 +244,13 @@ uint16_t const BATCH_SIZE = 11;
 
 namespace parameters_microphone
 {
+uint8_t const ARRAY_CHANNELS     = 5;
+uint8_t const ARRAY_TOP_LEFT     = 0;
+uint8_t const ARRAY_TOP_CENTER   = 1;
+uint8_t const ARRAY_TOP_RIGHT    = 2;
+uint8_t const ARRAY_BOTTOM_LEFT  = 3;
+uint8_t const ARRAY_BOTTOM_RIGHT = 4;
+
 uint32_t const SAMPLE_RATE    = 48000;
 uint8_t  const CHANNELS       = 2;
 uint16_t const GROUP_SIZE_RAW = 768;
@@ -954,6 +961,7 @@ class ipc_rc : public ipc
     void set_pv_iso_speed(uint32_t mode, uint32_t value);
     void set_pv_backlight_compensation(uint32_t state);
     void set_pv_scene_mode(uint32_t mode);
+    void set_flat_mode(uint32_t mode);
 };
 
 //------------------------------------------------------------------------------

extensions/client_matlab/+hl2ss/+mt/ipc_rc.m

@@ -78,5 +78,9 @@ function set_pv_white_balance_preset(self, preset)
     function set_pv_white_balance_value(self, value)
         self.module('ipc_call', self.port, 'set_pv_white_balance_value', uint32(value));
     end
+
+    function set_flat_mode(self, value)
+        self.module('ipc_call', self.port, 'set_flat_mode', uint32(value));
+    end
 end
 end

extensions/client_matlab/+hl2ss/parameters_microphone.m

@@ -1,6 +1,13 @@
 
 classdef parameters_microphone
 properties (Constant)
+    ARRAY_CHANNELS     = 5
+    ARRAY_TOP_LEFT     = 1
+    ARRAY_TOP_CENTER   = 2
+    ARRAY_TOP_RIGHT    = 3
+    ARRAY_BOTTOM_LEFT  = 4
+    ARRAY_BOTTOM_RIGHT = 5
+
     SAMPLE_RATE    = 48000;
     CHANNELS       = 2;
     GROUP_SIZE_RAW = 768;

extensions/client_matlab/hl2ss_matlab.cpp

@@ -480,7 +480,7 @@ class MexFunction : public matlab::mex::Function
         outputs[0] = std::move(o);
     }
 
-    void pack_microphone(int64_t frame_index, int32_t status, hl2ss::packet* packet, uint8_t profile, matlab::mex::ArgumentList outputs)
+    void pack_microphone(int64_t frame_index, int32_t status, hl2ss::packet* packet, uint8_t profile, uint8_t level, matlab::mex::ArgumentList outputs)
     {
         matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "audio" });
 
@@ -494,11 +494,15 @@ class MexFunction : public matlab::mex::Function
         {
         if (profile != hl2ss::audio_profile::RAW)
         {
-        o[0]["audio"]       = unpack_payload<float>(  packet->payload.get(), 0, packet->sz_payload, { hl2ss::parameters_microphone::CHANNELS, packet->sz_payload / (sizeof(float) * hl2ss::parameters_microphone::CHANNELS) });
+        o[0]["audio"]       = unpack_payload<float>(  packet->payload.get(), 0, packet->sz_payload, { hl2ss::parameters_microphone::CHANNELS,       packet->sz_payload / (sizeof(float)   * hl2ss::parameters_microphone::CHANNELS) });
+        }
+        else if (level == hl2ss::aac_level::L5)
+        {
+        o[0]["audio"]       = to_typed_array<float>(  packet->payload.get(),    packet->sz_payload, { hl2ss::parameters_microphone::ARRAY_CHANNELS, packet->sz_payload / (sizeof(float)   * hl2ss::parameters_microphone::ARRAY_CHANNELS) });
         }
         else
         {
-        o[0]["audio"]       = to_typed_array<int16_t>(packet->payload.get(),    packet->sz_payload, { hl2ss::parameters_microphone::CHANNELS, packet->sz_payload / (sizeof(int16_t) * hl2ss::parameters_microphone::CHANNELS) });
+        o[0]["audio"]       = to_typed_array<int16_t>(packet->payload.get(),    packet->sz_payload, { hl2ss::parameters_microphone::CHANNELS,       packet->sz_payload / (sizeof(int16_t) * hl2ss::parameters_microphone::CHANNELS) });
         }
         }
         }
@@ -688,7 +692,7 @@ class MexFunction : public matlab::mex::Function
         std::shared_ptr<hl2ss::packet> packet = grab(source, frame_index, status, inputs);
         hl2ss::rx_microphone const* p_rx = source->get_rx<hl2ss::rx_microphone>();
 
-        pack_microphone(frame_index, status, packet.get(), p_rx->profile, outputs);        
+        pack_microphone(frame_index, status, packet.get(), p_rx->profile, p_rx->level, outputs);
     }
 
     void get_packet_si(uint16_t port, matlab::mex::ArgumentList outputs, matlab::mex::ArgumentList inputs)
@@ -989,6 +993,11 @@ class MexFunction : public matlab::mex::Function
         uint32_t value = get_argument<uint32_t>(inputs);
         ipc_rc->set_pv_white_balance_value(value);
         }
+        else if (f == "set_flat_mode")
+        {
+        uint32_t value = get_argument<uint32_t>(inputs);
+        ipc_rc->set_flat_mode(value);
+        }
         else
         {
         throw std::runtime_error("Unknown method");

extensions/client_matlab/viewer/client_ipc_rc.m

@@ -26,6 +26,7 @@
 client.set_pv_video_temporal_denoising(hl2ss.pv_video_temporal_denoising_mode.Off);
 client.set_pv_white_balance_preset(hl2ss.pv_color_temperature_preset.Auto);
 client.set_pv_white_balance_value(hl2ss.pv_white_balance_value.Min);
+client.set_flat_mode(0)
 
 catch ME
     disp(ME.message);

extensions/client_matlab/viewer/client_stream_microphone_array.m

@@ -0,0 +1,49 @@
+%%
+% This script records microphone audio from the HoloLens and plays it.
+
+%% Settings
+
+% HoloLens address
+host = '192.168.1.7';
+
+% Recording length in seconds
+length = 10;
+
+% Channel
+% Options:
+% hl2ss.parameters_microphone.ARRAY_TOP_LEFT
+% hl2ss.parameters_microphone.ARRAY_TOP_CENTER
+% hl2ss.parameters_microphone.ARRAY_TOP_RIGHT
+% hl2ss.parameters_microphone.ARRAY_BOTTOM_LEFT
+% hl2ss.parameters_microphone.ARRAY_BOTTOM_RIGHT
+channel = hl2ss.parameters_microphone.ARRAY_TOP_LEFT;
+
+%%
+
+client = hl2ss.mt.sink_microphone(host, hl2ss.stream_port.MICROPHONE, 4096, hl2ss.audio_profile.RAW, hl2ss.aac_level.L5);
+client.open();
+
+packet_duration = hl2ss.parameters_microphone.GROUP_SIZE_RAW / hl2ss.parameters_microphone.SAMPLE_RATE;
+packet_count = length / packet_duration;
+record = [];
+frame_index = -1; % -1 for most recent frame
+
+try
+while (packet_count > 0)
+    data = client.get_packet_by_index(frame_index);
+    if (data.status == 0) % got packet
+        frame_index = data.frame_index + 1;
+        record = [record, data.audio];
+        packet_count = packet_count - 1;
+    else % no data
+        pause(packet_duration * 10); % wait for data
+    end
+end
+catch ME
+    disp(ME.message);
+end
+
+client.close();
+
+% play recorded sound
+sound(record(channel, :), hl2ss.parameters_microphone.SAMPLE_RATE);

hl2ss/hl2ss/Package.appxmanifest

@@ -9,7 +9,7 @@
   <Identity
     Name="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8"
     Publisher="CN=jcds"
-    Version="1.0.26.0" />
+    Version="1.0.27.0" />
   <mp:PhoneIdentity PhoneProductId="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8" PhonePublisherId="00000000-0000-0000-0000-000000000000"/>
   <Properties>
     <DisplayName>hl2ss</DisplayName>

hl2ss/hl2ss/extended_execution.cpp

@@ -6,16 +6,20 @@
 #include <winrt/Windows.ApplicationModel.h>
 #include <winrt/Windows.ApplicationModel.ExtendedExecution.h>
 #include <winrt/Windows.ApplicationModel.ExtendedExecution.Foreground.h>
+#include <winrt/Windows.Storage.h>
 
 using namespace winrt::Windows::Foundation;
 using namespace winrt::Windows::ApplicationModel;
 using namespace winrt::Windows::ApplicationModel::ExtendedExecution;
 using namespace winrt::Windows::ApplicationModel::ExtendedExecution::Foreground;
+using namespace winrt::Windows::Storage;
 
 //-----------------------------------------------------------------------------
 // Global Variables
 //-----------------------------------------------------------------------------
 
+static wchar_t const* g_flat_name = L"flat_mode.cfg";
+
 static ExtendedExecutionForegroundSession g_eefs = nullptr;
 static bool g_status = false;
 
@@ -49,3 +53,44 @@ bool ExtendedExecution_Status()
 {
 	return g_status;
 }
+
+// OK
+void ExtendedExecution_SetFlatMode(bool flat)
+{
+	StorageFolder folder = ApplicationData::Current().LocalFolder();
+	winrt::hstring name{ g_flat_name };
+
+	try
+	{
+		if (flat)
+		{
+			folder.CreateFileAsync(name).get();
+		}
+		else
+		{
+			StorageFile file = folder.GetFileAsync(name).get();
+			file.DeleteAsync().get();
+		}
+	}
+	catch (...)
+	{
+	}
+}
+
+// OK
+bool ExtendedExecution_GetFlatMode()
+{
+	StorageFolder folder = ApplicationData::Current().LocalFolder();
+	winrt::hstring name{ g_flat_name };
+
+	try
+	{
+		folder.GetFileAsync(name).get();
+		return true;
+	}
+	catch (...)
+	{
+	}
+
+	return false;
+}

hl2ss/hl2ss/extended_execution.h

@@ -3,3 +3,5 @@
 
 void ExtendedExecution_Request();
 bool ExtendedExecution_Status();
+void ExtendedExecution_SetFlatMode(bool flat);
+bool ExtendedExecution_GetFlatMode();

hl2ss/hl2ss/ipc_rc.cpp

@@ -4,6 +4,7 @@
 #include "log.h"
 #include "holographic_space.h"
 #include "personal_video.h"
+#include "extended_execution.h"
 #include "timestamps.h"
 #include "nfo.h"
 
@@ -290,6 +291,22 @@ static void RC_MSG_SetPVSceneMode(SOCKET clientsocket)
     PersonalVideo_SetSceneMode(mode);
 }
 
+// OK
+static void RC_MSG_SetFlatMode(SOCKET clientsocket)
+{
+    bool ok;
+    uint32_t mode;
+
+    ok = recv_u32(clientsocket, mode);
+    if (!ok)
+    {
+        RC_TransferError();
+        return;
+    }
+
+    ExtendedExecution_SetFlatMode(mode != 0);
+}
+
 // OK
 static void RC_Dispatch(SOCKET clientsocket)
 {
@@ -318,6 +335,7 @@ static void RC_Dispatch(SOCKET clientsocket)
     case 0x0A: RC_MSG_SetPVIsoSpeed(clientsocket);               break;
     case 0x0B: RC_MSG_SetPVBacklightCompensation(clientsocket);  break;
     case 0x0C: RC_MSG_SetPVSceneMode(clientsocket);              break;
+    case 0x0D: RC_MSG_SetFlatMode(clientsocket);                 break;
     default:
         RC_TransferError();
         return;

hl2ss/hl2ss/main.cpp

@@ -34,6 +34,7 @@ struct App : winrt::implements<App, IFrameworkViewSource, IFrameworkView>
 {
 	bool m_windowClosed = false;
 	bool m_init = false;
+	bool m_flat = false;
 
 	IFrameworkView CreateView()
 	{
@@ -56,6 +57,8 @@ struct App : winrt::implements<App, IFrameworkViewSource, IFrameworkView>
 
 		InitializeSockets();
 		MFStartup(MF_VERSION);
+
+		m_flat = ExtendedExecution_GetFlatMode();
 	}
 
 	void Load(winrt::hstring const&)
@@ -72,9 +75,11 @@ struct App : winrt::implements<App, IFrameworkViewSource, IFrameworkView>
 
 		if (m_init) { return; }
 
-#ifndef HL2SS_FLAT 
+		if (!m_flat)
+		{
 		HolographicSpace_Initialize();
-#endif
+		}
+
 		Locator_Initialize();
 		ResearchMode_Initialize();
 		SpatialInput_Initialize();
@@ -108,12 +113,13 @@ struct App : winrt::implements<App, IFrameworkViewSource, IFrameworkView>
 		{
 		window.Dispatcher().ProcessEvents(CoreProcessEventsOption::ProcessAllIfPresent);
 
-#ifndef HL2SS_FLAT
+		if (!m_flat)
+		{
 		HolographicSpace_Update();
 		HolographicSpace_Clear();
 		// Draw
 		HolographicSpace_Present();
-#endif
+		}
 		}
 
 		CoreApplication::Exit();