diff --git a/MonoGame.Framework/Media/Song.cs b/MonoGame.Framework/Media/Song.cs index 740ff708166..08292303c77 100644 --- a/MonoGame.Framework/Media/Song.cs +++ b/MonoGame.Framework/Media/Song.cs @@ -171,7 +171,7 @@ public override bool Equals(Object obj) /// public TimeSpan Duration { - get { return PlatformGetDuration(); } + get { return _duration; } } /// diff --git a/MonoGame.Framework/Platform/Native/Audio.Interop.cs b/MonoGame.Framework/Platform/Native/Audio.Interop.cs index f9fd08b2af4..a0a44562bad 100644 --- a/MonoGame.Framework/Platform/Native/Audio.Interop.cs +++ b/MonoGame.Framework/Platform/Native/Audio.Interop.cs @@ -140,7 +140,7 @@ public static partial void Buffer_InitializeXact( #region Voice [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Create", StringMarshalling = StringMarshalling.Utf8)] - public static partial MGA_Voice* Voice_Create(MGA_System* system); + public static partial MGA_Voice* Voice_Create(MGA_System* system, int sampleRate = 0, int channels = 0); [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Destroy", StringMarshalling = StringMarshalling.Utf8)] public static partial void Voice_Destroy(MGA_Voice* voice); @@ -152,7 +152,7 @@ public static partial void Buffer_InitializeXact( public static partial void Voice_SetBuffer(MGA_Voice* voice, MGA_Buffer* buffer); [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_AppendBuffer", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Voice_AppendBuffer(MGA_Voice* voice, byte[] buffer, int offset, int count, [MarshalAs(UnmanagedType.U1)] bool clear); + public static partial void Voice_AppendBuffer(MGA_Voice* voice, byte* buffer, uint size); [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Play", StringMarshalling = StringMarshalling.Utf8)] public static partial void Voice_Play(MGA_Voice* voice, [MarshalAs(UnmanagedType.U1)] bool looped); @@ -169,6 +169,9 @@ public static partial void Buffer_InitializeXact( [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_GetState", StringMarshalling = StringMarshalling.Utf8)] public static partial SoundState Voice_GetState(MGA_Voice* voice); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_GetPosition", StringMarshalling = StringMarshalling.Utf8)] + public static partial ulong Voice_GetPosition(MGA_Voice* voice); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_SetPan", StringMarshalling = StringMarshalling.Utf8)] public static partial void Voice_SetPan(MGA_Voice* voice, float pan); diff --git a/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs b/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs index 2270927e7c1..6625148ada5 100644 --- a/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs +++ b/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs @@ -11,7 +11,7 @@ public sealed partial class DynamicSoundEffectInstance : SoundEffectInstance { private unsafe void PlatformCreate() { - Voice = MGA.Voice_Create(SoundEffect.System); + Voice = MGA.Voice_Create(SoundEffect.System, _sampleRate, (int)_channels); } private unsafe int PlatformGetPendingBufferCount() @@ -49,7 +49,10 @@ private unsafe void PlatformStop() private unsafe void PlatformSubmitBuffer(byte[] buffer, int offset, int count) { if (Voice != null) - MGA.Voice_AppendBuffer(Voice, buffer, offset, count, false); + { + fixed (byte* ptr = buffer) + MGA.Voice_AppendBuffer(Voice, ptr + offset, (uint)count); + } } private unsafe void PlatformDispose(bool disposing) diff --git a/MonoGame.Framework/Platform/Native/Media.Interop.cs b/MonoGame.Framework/Platform/Native/Media.Interop.cs index bda979ff72d..a9b0cb510bc 100644 --- a/MonoGame.Framework/Platform/Native/Media.Interop.cs +++ b/MonoGame.Framework/Platform/Native/Media.Interop.cs @@ -4,93 +4,122 @@ using System; using System.Runtime.InteropServices; +using Microsoft.Xna.Framework; +using Microsoft.Xna.Framework.Audio; using Microsoft.Xna.Framework.Media; namespace MonoGame.Interop; [MGHandle] -internal readonly struct MGM_Song { } +internal readonly struct MGM_AudioDecoder{ } [MGHandle] -internal readonly struct MGM_Video { } +internal readonly struct MGM_VideoDecoder { } - -internal static unsafe partial class MGM +struct MGM_AudioDecoderInfo { - #region Song - - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - public delegate void NativeFinishedCallback(nint callbackData); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Create", StringMarshalling = StringMarshalling.Utf8)] - public static partial MGM_Song* Song_Create(string mediaFilePath); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Destroy", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_Destroy(MGM_Song* song); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_GetDuration", StringMarshalling = StringMarshalling.Utf8)] - public static partial ulong Song_GetDuration(MGM_Song* song); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_GetPosition", StringMarshalling = StringMarshalling.Utf8)] - public static partial ulong Song_GetPosition(MGM_Song* song); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_GetVolume", StringMarshalling = StringMarshalling.Utf8)] - public static partial float Song_GetVolume(MGM_Song* song); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_SetVolume", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_SetVolume(MGM_Song* song, float volume); + /// + /// The audio samples per second, per channel. + /// + public int samplerate; + + /// + /// The number of audio channels (typically 1 or 2). + /// + public int channels; + + /// + /// The estimated duration of the audio in milliseconds. + /// + public ulong duration; +} - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Play", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_Play(MGM_Song* song, ulong startPositionMs, [MarshalAs(UnmanagedType.FunctionPtr)] NativeFinishedCallback callback, nint callbackData); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Pause", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_Pause(MGM_Song* song); +struct MGM_VideoDecoderInfo +{ + /// + /// The width of the video frames. + /// + public int width; + + /// + /// The height of the video frames. + /// + public int height; + + /// + /// The number of frames per second. + /// + public float fps; + + /// + /// The estimated duration of the video in milliseconds. + /// + public ulong duration; +} - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Resume", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_Resume(MGM_Song* song); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Song_Stop", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Song_Stop(MGM_Song* song); +internal static unsafe partial class MGM +{ + #region Audio Decoder + + /// + /// This returns an audio decoder that returns a stream of PCM data from an audio file. + /// + /// The absolute file path to the audio file. + /// Returns information about the opened audio file. + /// Returns the audio decoder ready to read data or null if the format is unsupported. + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_AudioDecoder_Create", StringMarshalling = StringMarshalling.Utf8)] + public static partial MGM_AudioDecoder* AudioDecoder_Create(string filepath, out MGM_AudioDecoderInfo info); + + /// + /// This releases all internal resources, closes the file, and destroys the audio decoder. + /// + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_AudioDecoder_Destroy", StringMarshalling = StringMarshalling.Utf8)] + public static partial void AudioDecoder_Destroy(MGM_AudioDecoder* decoder); + + /// + /// Set the position of the audio decoder in milliseconds. + /// + /// The decoder. + /// The time in millseconds from the start of the audio file. + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_AudioDecoder_SetPosition", StringMarshalling = StringMarshalling.Utf8)] + public static partial void AudioDecoder_SetPosition(MGM_AudioDecoder* decoder, ulong timeMS); + + /// + /// Decode some PCM data from the audio file. + /// + /// + /// + /// + /// Returns true if we've reached the end of the file. + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_AudioDecoder_Decode", StringMarshalling = StringMarshalling.Utf8)] + [return:MarshalAs(UnmanagedType.U1)] + public static partial bool AudioDecoder_Decode(MGM_AudioDecoder* decoder, out byte* buffer, out uint size); #endregion #region Video - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Create", StringMarshalling = StringMarshalling.Utf8)] - public static partial MGM_Video* Video_Create(string mediaFilePath, int cachedFrameNum, out int width, out int height, out float fps, out ulong duration); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Destroy", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_Destroy(MGM_Video* video); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_GetState", StringMarshalling = StringMarshalling.Utf8)] - public static partial MediaState Video_GetState(MGM_Video* video); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_GetPosition", StringMarshalling = StringMarshalling.Utf8)] - public static partial ulong Video_GetPosition(MGM_Video* video); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_SetVolume", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_SetVolume(MGM_Video* video, float volume); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_SetLooped", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_SetLooped(MGM_Video* video, [MarshalAs(UnmanagedType.U1)] bool looped); - - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Play", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_Play(MGM_Video* video); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_Create", StringMarshalling = StringMarshalling.Utf8)] + public static partial MGM_VideoDecoder* VideoDecoder_Create(MGG_GraphicsDevice* device, string filepath, out MGM_VideoDecoderInfo info); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Pause", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_Pause(MGM_Video* video); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_Destroy", StringMarshalling = StringMarshalling.Utf8)] + public static partial void VideoDecoder_Destroy(MGM_VideoDecoder* decoder); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Resume", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_Resume(MGM_Video* video); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_GetAudioDecoder", StringMarshalling = StringMarshalling.Utf8)] + public static partial MGM_AudioDecoder* VideoDecoder_GetAudioDecoder(MGM_VideoDecoder* decoder, out MGM_AudioDecoderInfo info); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_Stop", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_Stop(MGM_Video* video); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_GetPosition", StringMarshalling = StringMarshalling.Utf8)] + public static partial ulong VideoDecoder_GetPosition(MGM_VideoDecoder* decoder); - [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_Video_GetFrame", StringMarshalling = StringMarshalling.Utf8)] - public static partial void Video_GetFrame(MGM_Video* video, out uint frame, out MGG_Texture* handle); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_SetLooped", StringMarshalling = StringMarshalling.Utf8)] + public static partial void VideoDecoder_SetLooped(MGM_VideoDecoder* decoder, [MarshalAs(UnmanagedType.U1)] bool looped); + [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGM_VideoDecoder_Decode", StringMarshalling = StringMarshalling.Utf8)] + public static partial MGG_Texture* VideoDecoder_Decode(MGM_VideoDecoder* decoder); #endregion } diff --git a/MonoGame.Framework/Platform/Native/Song.Native.cs b/MonoGame.Framework/Platform/Native/Song.Native.cs index 34bea91c54d..77f7e52b6d1 100644 --- a/MonoGame.Framework/Platform/Native/Song.Native.cs +++ b/MonoGame.Framework/Platform/Native/Song.Native.cs @@ -3,33 +3,102 @@ // file 'LICENSE.txt', which is part of this source code package. using System; -using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Xna.Framework.Audio; using MonoGame.Interop; + namespace Microsoft.Xna.Framework.Media; public sealed partial class Song : IEquatable, IDisposable { - private unsafe MGM_Song* _song; + private unsafe MGM_AudioDecoder* _decoder; + private unsafe MGA_Voice* _voice; + + private MGM_AudioDecoderInfo _info; + + private readonly ManualResetEvent _stop = new ManualResetEvent(false); + private Thread _thread; - private GCHandle _self; + private float _volume = 1.0f; + + private unsafe void DecoderStream() + { + bool start_voice = true; + bool finished = false; + + Console.WriteLine("DecoderStream"); + + while (true) + { + // Do we need to stop? + if (_stop.WaitOne(0)) + break; + + var count = MGA.Voice_GetBufferCount(_voice); + if (count > 2) + { + // TODO: This sucks... add OnBufferEnd type of callback + // into the voice API so we don't have useless sleeps. + Thread.Sleep(100); + continue; + } + + finished = MGM.AudioDecoder_Decode(_decoder, out var buffer, out var size); + + if (size > 0) + { + MGA.Voice_AppendBuffer(_voice, buffer, size); + + if (start_voice) + { + MGA.Voice_Play(_voice, false); + start_voice = false; + } + } + + if (finished) + { + // Signal on the main thread. + Threading.OnUIThread(() => DonePlaying(this, EventArgs.Empty)); + break; + } + } + + // We're done streaming. + } #region The playback API used by MediaPlayer private unsafe void PlatformInitialize(string fileName) { - _song = MGM.Song_Create(fileName); + var absolutePath = MGP.Platform_MakePath(TitleContainer.Location, fileName); + + _decoder = MGM.AudioDecoder_Create(absolutePath, out _info); + if (_decoder == null) + return; + + SoundEffect.Initialize(); + + _voice = MGA.Voice_Create(SoundEffect.System, _info.samplerate, _info.channels); + + _duration = TimeSpan.FromMilliseconds(_info.duration); } private unsafe void PlatformDispose(bool disposing) { - if (_self.IsAllocated) - _self.Free(); + Stop(); + + if (_voice != null) + { + MGA.Voice_Destroy(_voice); + _voice = null; + } - if (_song != null) + if (_decoder != null) { - MGM.Song_Destroy(_song); - _song = null; + MGM.AudioDecoder_Destroy(_decoder); + _decoder = null; } } @@ -38,29 +107,19 @@ private int PlatformGetPlayCount() return _playCount; } - private unsafe TimeSpan PlatformGetDuration() - { - if (_song == null) - return TimeSpan.Zero; - - var milliseconds = MGM.Song_GetDuration(_song); - return TimeSpan.FromMilliseconds(milliseconds); - } - internal unsafe float Volume { get { - if (_song == null) - return 0.0f; - - return MGM.Song_GetVolume(_song); + return _volume; } set { - if (_song != null) - MGM.Song_SetVolume(_song, value); + _volume = value; + + if (_voice != null) + MGA.Voice_SetVolume(_voice, _volume); } } @@ -68,34 +127,17 @@ internal unsafe TimeSpan Position { get { - if (_song == null) + if (_voice == null) return TimeSpan.Zero; - var milliseconds = MGM.Song_GetPosition(_song); + var milliseconds = MGA.Voice_GetPosition(_voice); return TimeSpan.FromMilliseconds(milliseconds); } } - internal static void FinishedCallback(nint callbackData) - { - var self = GCHandle.FromIntPtr(callbackData); - var song = self.Target as Song; - - // This could happen if we were disposed. - if (song == null) - return; - - // This callback is likely coming from a platform - // specific native thread. So queue the event to - // the main game thread for processing on the - // next tick. - - Threading.OnUIThread(() => song.DonePlaying(song, EventArgs.Empty)); - } - internal unsafe void Play(TimeSpan? startPosition, FinishedPlayingHandler handler) { - if (_song == null) + if (_decoder == null) return; ulong milliseconds = 0; @@ -105,36 +147,49 @@ internal unsafe void Play(TimeSpan? startPosition, FinishedPlayingHandler handle // Only setup the finished callback once. if (DonePlaying == null) DonePlaying += handler; - if (!_self.IsAllocated) - _self = GCHandle.Alloc(this, GCHandleType.Weak); - MGM.Song_Play(_song, milliseconds, FinishedCallback, (nint)_self); + // Stop the current playback which cleans stuff up. + Stop(); + + // Move the decoder to the new position. + MGM.AudioDecoder_SetPosition(_decoder, milliseconds); + + // The thread does the rest of the work. + _stop.Reset(); + _thread = new Thread(DecoderStream); + _thread.Start(); _playCount++; } internal unsafe void Pause() { - if (_song == null) + if (_voice == null) return; - MGM.Song_Pause(_song); + // The thread will stop processing on its own. + MGA.Voice_Pause(_voice); } internal unsafe void Resume() { - if (_song == null) + if (_voice == null) return; - MGM.Song_Resume(_song); + MGA.Voice_Resume(_voice); } internal unsafe void Stop() { - if (_song == null) + if (_thread == null) return; - MGM.Song_Stop(_song); + MGA.Voice_Stop(_voice, false); + + // Halt the thread. + _stop.Set(); + _thread.Join(); + _thread = null; } diff --git a/MonoGame.Framework/Platform/Native/Video.Native.cs b/MonoGame.Framework/Platform/Native/Video.Native.cs index 139a4df7d38..62c2440daad 100644 --- a/MonoGame.Framework/Platform/Native/Video.Native.cs +++ b/MonoGame.Framework/Platform/Native/Video.Native.cs @@ -3,40 +3,183 @@ // file 'LICENSE.txt', which is part of this source code package. using System; -using System.IO; -using MonoGame.Interop; +using System.Threading; +using System.Collections.Concurrent; +using Microsoft.Xna.Framework.Audio; using Microsoft.Xna.Framework.Graphics; +using MonoGame.Interop; namespace Microsoft.Xna.Framework.Media; public sealed partial class Video : IDisposable { - private unsafe MGM_Video* _video; - private Texture2D[] _frameCache; + // TODO: This entire class is untested and speculative. It + // is based on the audio decoder design. + // + // We need to add support for one video codec (likely Theora) + // to help prove this API and make adjustments to it. + // + // Things i expect we could have issues with: + // + // - Video getting out of sync. + // - Too much CPU use. + // - For sure we have race conditions. + // + + private unsafe MGM_VideoDecoder* _decoderV; + private MGM_VideoDecoderInfo _infoV; + + private unsafe MGM_AudioDecoder* _decoderA; + private MGM_AudioDecoderInfo _infoA; + + private unsafe MGA_Voice* _voice; + + private ManualResetEvent _paused = new ManualResetEvent(false); + private int _state; + private Thread _thread; + + private ConcurrentQueue _frames; + private bool _muted = false; private float _volume = 1.0f; private bool _looped = false; + private unsafe void DecoderStream() + { + bool play_voice = true; + + Console.WriteLine("Video.DecoderStream"); + + // Create the voice if we have audio data. + if (_decoderA != null) + _voice = MGA.Voice_Create(SoundEffect.System, _infoA.samplerate, _infoA.channels); + + bool videoFinished = false; + bool audioFinished = false; + + MGG_Texture* lastFrame = null; + + var device = Game.Instance.GraphicsDevice; + + var sleep = (int)(1000.0f / _infoV.fps); + + while (true) + { + // Get the current state. + var state = (MediaState)Interlocked.CompareExchange(ref _state, -1, -1); + + // Do we need to stop playback? + if (state == MediaState.Stopped) + break; + + // Are we paused? + if (state == MediaState.Paused) + { + if (_voice != null) + { + MGA.Voice_Pause(_voice); + play_voice = true; + } + + // Block waiting for it to be signaled. + _paused.Reset(); + _paused.WaitOne(); + continue; + } + + // Process an audio frame. + if (_voice != null && !audioFinished) + { + var count = MGA.Voice_GetBufferCount(_voice); + if (count < 3) + { + MGM.AudioDecoder_Decode(_decoderA, out var buffer, out var size); + + if (size > 0) + MGA.Voice_AppendBuffer(_voice, buffer, size); + else + audioFinished = true; + } + + if (play_voice) + { + MGA.Voice_Play(_voice, false); + play_voice = false; + } + } + + // Process video frames. + if (!videoFinished) + { + var frame = MGM.VideoDecoder_Decode(_decoderV); + + // Did we get a new frame? + if (frame != lastFrame) + { + // A null frame means we're out of video data. + if (frame == null) + videoFinished = true; + else + { + // Queue the frame for rendering. + + lastFrame = frame; + + var texture = new Texture2D( + device, + frame, + _infoV.width, + _infoV.height, + false, + SurfaceFormat.Color, + Texture2D.SurfaceType.Texture, + 1); + + _frames.Enqueue(texture); + } + } + } + + // If both audio and video are finished then exit. + if (videoFinished && audioFinished) + break; + + // TODO: We could maybe do better? + Thread.Sleep(sleep); + } + + // We're done streaming.. cleanup. + if (_voice != null) + { + MGA.Voice_Destroy(_voice); + _voice = null; + } + + Interlocked.Exchange(ref _state, (int)MediaState.Stopped); + } + private unsafe void PlatformInitialize() { - var mediaFilePath = Path.Combine(TitleContainer.Location, FileName); + var absolutePath = MGP.Platform_MakePath(TitleContainer.Location, FileName); + + // TODO: Maybe there is a better place to get this + // that doesn't assume Game exists. + var device = Game.Instance.GraphicsDevice; - // TODO: We should expose the back buffer count so that - // video playback and optimize the number of textures created - // for video frames. - int cachedFrameNum = 2; // Game.Instance.GraphicsDevice.BackBufferCount; + _decoderV = MGM.VideoDecoder_Create(device.Handle, absolutePath, out _infoV); + if (_decoderV == null) + return; - // TODO: May be worth letting the video return texture - // format, so it can avoid CPU conversions when possible. + Width = _infoV.width; + Height = _infoV.height; + FramesPerSecond = _infoV.fps; + Duration = TimeSpan.FromMilliseconds(_infoV.duration); - _video = MGM.Video_Create(mediaFilePath, cachedFrameNum, out var width, out var height, out var fps, out var duration); + _state = (int)MediaState.Stopped; - Width = width; - Height = height; - Duration = TimeSpan.FromMilliseconds(duration); - FramesPerSecond = fps; - _frameCache = new Texture2D[cachedFrameNum]; + // Get the audio decoder if we have one. + _decoderA = MGM.VideoDecoder_GetAudioDecoder(_decoderV, out _infoA); // Unsupported VideoSoundtrackType = VideoSoundtrackType.MusicAndDialog; @@ -44,49 +187,42 @@ private unsafe void PlatformInitialize() private unsafe void PlatformDispose(bool disposing) { - if (_video != null) + Stop(); + + if (_decoderV != null) { - MGM.Video_Destroy(_video); - _video = null; + MGM.VideoDecoder_Destroy(_decoderV); + _decoderV = null; + _decoderA = null; } } internal unsafe Texture2D GetTexture() { - if (_video == null) + if (_decoderV == null) return null; - MGM.Video_GetFrame(_video, out uint frame, out MGG_Texture* handle); - - uint index = frame % (uint)_frameCache.Length; - var texture = _frameCache[index]; - - if (texture == null) - { - // The video player owns the texture handle here. - - _frameCache[index] = texture = new Texture2D( - Game.Instance.GraphicsDevice, - handle, - Width, - Height, - false, - SurfaceFormat.Color, - Texture2D.SurfaceType.Texture, - 1); - } + // If we have multiple queued frames then we're behind real time + // and need to pop off the texture to display. + // + // If we have only one frame then keep it in the queue. This is + // for cases like pausing, rendering between frames, or the last + // frame of a video. + // + Texture2D texture = null; + if (_frames.Count > 1) + _frames.TryDequeue(out texture); + else + _frames.TryPeek(out texture); return texture; } - internal unsafe MediaState State + internal MediaState State { get { - if (_video == null) - return MediaState.Stopped; - - return MGM.Video_GetState(_video); + return (MediaState)Interlocked.CompareExchange(ref _state, -1, -1); } } @@ -94,10 +230,10 @@ internal unsafe TimeSpan Position { get { - if (_video == null) + if (_decoderV == null) return TimeSpan.Zero; - var position = MGM.Video_GetPosition(_video); + var position = MGM.VideoDecoder_GetPosition(_decoderV); return TimeSpan.FromMilliseconds(position); } } @@ -107,8 +243,8 @@ internal unsafe bool IsLooped set { _looped = value; - if (_video != null) - MGM.Video_SetLooped(_video, value); + if (_decoderV != null) + MGM.VideoDecoder_SetLooped(_decoderV, value); } get @@ -124,14 +260,14 @@ internal unsafe bool IsMuted if (value) { _muted = true; - if (_video != null) - MGM.Video_SetVolume(_video, 0.0f); + if (_voice != null) + MGA.Voice_SetVolume(_voice, 0.0f); } else { _muted = false; - if (_video != null) - MGM.Video_SetVolume(_video, _volume); + if (_voice != null) + MGA.Voice_SetVolume(_voice, _volume); } } @@ -146,8 +282,8 @@ internal unsafe float Volume set { _volume = value; - if (!_muted && _video != null) - MGM.Video_SetVolume(_video, _volume); + if (!_muted && _voice != null) + MGA.Voice_SetVolume(_voice, _volume); } get @@ -158,25 +294,56 @@ internal unsafe float Volume internal unsafe void Play() { - if (_video != null) - MGM.Video_Play(_video); + if (_decoderV == null) + return; + + // Stop the current playback which cleans stuff up. + Stop(); + + // The thread does the work. + _state = (int)MediaState.Playing; + _thread = new Thread(DecoderStream); + _thread.Start(); } internal unsafe void Pause() { - if (_video != null) - MGM.Video_Pause(_video); + if (_decoderV == null) + return; + + // Nothing to do if we're not playing. + if (State != MediaState.Playing) + return; + + Interlocked.CompareExchange(ref _state, (int)MediaState.Paused, (int)MediaState.Playing); } internal unsafe void Resume() { - if (_video != null) - MGM.Video_Resume(_video); + if (_decoderV == null) + return; + + // Nothing to do if we're not paused. + if (State != MediaState.Paused) + return; + + // If we're paused then go back to playing state and wake up the thread. + Interlocked.Exchange(ref _state, (int)MediaState.Playing); + _paused.Set(); } internal unsafe void Stop() { - if (_video != null) - MGM.Video_Stop(_video); + // Nothing to do if we're stopped already. + if (State == MediaState.Stopped) + return; + + // Signal the thread to stop... signal just in case it is paused. + Interlocked.Exchange(ref _state, (int)MediaState.Stopped); + _paused.Set(); + + // Wait for the thread. + _thread.Join(); + _thread = null; } } diff --git a/native/monogame/common/MGM.cpp b/native/monogame/common/MGM.cpp index d17ad0184ee..2774d410012 100644 --- a/native/monogame/common/MGM.cpp +++ b/native/monogame/common/MGM.cpp @@ -6,180 +6,174 @@ struct MGG_Texture; #include "api_MGM.h" -#include "mg_common.h" +#include "MGM_common.h" +#include "api_MGA.h" +#include "api_MGG.h" +#include -struct MGM_Song -{ - mgulong duration; - - void (*finishCallback)(void*); - void* finishData; -}; -MGM_Song* MGM_Song_Create(const char* mediaFilePath) +void MGM_ReadSignature(const char* filepath, MGM_SIGNATURE) { - assert(mediaFilePath != nullptr); + memset(signature, 0, 16); - // TODO: This should detect the media format using - // standard native format decoder libraries that are - // portable to all our target platforms: - // - // libvorbis - // minimp3 - // wave ?? - // mp4 ?? - // FLAC ?? - // - // It should then spin up thread which decodes the - // audio and streams buffers to a native SoundEffect. - - auto song = new MGM_Song(); - song->duration = 0; - song->finishCallback = nullptr; - song->finishData = nullptr; - return song; -} + FILE* handle = fopen(filepath, "rb"); + if (handle == nullptr) + return; -mgulong MGM_Song_GetDuration(MGM_Song* song) -{ - assert(song != nullptr); - return song->duration; + fread(signature, 1, 16, handle); + fclose(handle); } -mgulong MGM_Song_GetPosition(MGM_Song* song) +MGM_AudioDecoder* MGM_AudioDecoder_TryCreate_Ogg(MGM_SIGNATURE) { - assert(song != nullptr); - return 0; + // TODO: Implement me! + // + // - This should be moved into its own CPP. + // - We need to add Ogg support to native build. + // - How do we compile Ogg for consoles? + // + return nullptr; } -mgfloat MGM_Song_GetVolume(MGM_Song* song) +MGM_AudioDecoder* MGM_AudioDecoder_TryCreate_Mp3(MGM_SIGNATURE) { - assert(song != nullptr); - return 0; + // TODO: Implement me! + // + // - This should be moved into its own CPP. + // - Should we use a single header mp3 decoder? + // + return nullptr; } -void MGM_Song_SetVolume(MGM_Song* song, mgfloat volume) -{ - assert(song != nullptr); -} +#if defined(_WIN32) -void MGM_Song_Pause(MGM_Song* song) +MGM_AudioDecoder* MGM_AudioDecoder_Create(const char* filepath, MGM_AudioDecoderInfo& info) { - assert(song != nullptr); -} + assert(filepath != nullptr); -void MGM_Song_Play(MGM_Song* song, mgulong startPositionMs, void (*callback)(void*), void* callbackData) -{ - assert(song != nullptr); + MGM_SIGNATURE; + MGM_ReadSignature(filepath, signature); - song->finishCallback = callback; - song->finishData = callbackData; + // Try the common decoders. + MGM_AudioDecoder* decoder = nullptr; + decoder = decoder ? decoder : MGM_AudioDecoder_TryCreate_Ogg(signature); + decoder = decoder ? decoder : MGM_AudioDecoder_TryCreate_Mp3(signature); + + if (decoder == nullptr) + { + info.samplerate = 0; + info.channels = 0; + info.duration = 0; + return nullptr; + } + + decoder->Initialize(filepath, info); + return decoder; } -void MGM_Song_Resume(MGM_Song* song) +#endif + +void MGM_AudioDecoder_Destroy(MGM_AudioDecoder* decoder) { - assert(song != nullptr); + assert(decoder != nullptr); + delete decoder; } -void MGM_Song_Stop(MGM_Song* song) +void MGM_AudioDecoder_SetPosition(MGM_AudioDecoder* decoder, mgulong timeMS) { - assert(song != nullptr); + assert(decoder != nullptr); + decoder->SetPosition(timeMS); } -void MGM_Song_Destroy(MGM_Song* song) +mgbool MGM_AudioDecoder_Decode(MGM_AudioDecoder* decoder, mgbyte*& buffer, mguint& size) { - assert(song != nullptr); - delete song; + assert(decoder != nullptr); + return decoder->Decode(buffer, size); } -struct MGM_Video -{ - mguint width; - mguint height; - mgfloat fps; - mgulong duration; - mgint cachedFrames; -}; - -MGM_Video* MGM_Video_Create(const char* mediaFilePath, mgint cachedFrameNum, mgint& width, mgint& height, mgfloat& fps, mgulong& duration) -{ - assert(mediaFilePath != nullptr); - // TODO: Like Song above we should detect the media - // format from a native decoder libraries that are - // portable to all our target platforms. +MGM_VideoDecoder* MGM_VideoDecoder_TryCreate_Theora(MGM_SIGNATURE) +{ + // TODO: Implement me! // - // It should then spin up thread which decodes the - // video/audio streams. - - // TOOD: Ideally we just support OpenH264+AAC which is pretty - // much industry standard now. Anything else is not - // importaint unless a new standard comes around. - - auto video = new MGM_Video(); - video->duration = duration = 0; - video->width = width = 0; - video->height = height = 0; - video->fps = fps = 0.0f; - video->cachedFrames = cachedFrameNum; - - return video; + // - This should be moved into its own CPP. + // - We need to add Theora support to native build. + // - How do we compile Theora for consoles? + // + return nullptr; } -void MGM_Video_Destroy(MGM_Video* video) +MGM_VideoDecoder* MGM_VideoDecoder_TryCreate_OpenH264(MGM_SIGNATURE) { - assert(video != nullptr); - delete video; + // TODO: Implement me! + // + // See https://github.com/cisco/openh264 + // + // - This should be moved into its own CPP. + // - We need to add lib to native build. + // - How do we compile lib for consoles? + // + return nullptr; } -MGMediaState MGM_Video_GetState(MGM_Video* video) -{ - assert(video != nullptr); - return MGMediaState::Stopped; -} +#if defined(_WIN32) -mgulong MGM_Video_GetPosition(MGM_Video* video) +MGM_VideoDecoder* MGM_VideoDecoder_Create(MGG_GraphicsDevice* device, const char* filepath, MGM_VideoDecoderInfo& info) { - assert(video != nullptr); - return 0; -} + assert(filepath != nullptr); -void MGM_Video_SetVolume(MGM_Video* video, mgfloat volume) -{ - assert(video != nullptr); -} + MGM_SIGNATURE; + MGM_ReadSignature(filepath, signature); -void MGM_Video_SetLooped(MGM_Video* video, mgbool looped) -{ - assert(video != nullptr); + // Try the common decoders. + MGM_VideoDecoder* decoder = nullptr; + decoder = decoder ? decoder : MGM_VideoDecoder_TryCreate_Theora(signature); + decoder = decoder ? decoder : MGM_VideoDecoder_TryCreate_OpenH264(signature); + + if (decoder == nullptr) + { + info.width = 0; + info.height = 0; + info.fps = 0; + info.duration = 0; + return nullptr; + } + + decoder->Initialize(filepath, info); + return decoder; } -void MGM_Video_Play(MGM_Video* video) +#endif + +void MGM_VideoDecoder_Destroy(MGM_VideoDecoder* decoder) { - assert(video != nullptr); + assert(decoder != nullptr); + delete decoder; } -void MGM_Video_Pause(MGM_Video* video) +MGM_AudioDecoder* MGM_VideoDecoder_GetAudioDecoder(MGM_VideoDecoder* decoder, MGM_AudioDecoderInfo& info) { - assert(video != nullptr); + assert(decoder != nullptr); + return decoder->GetAudioDecoder(info); } -void MGM_Video_Resume(MGM_Video* video) +mgulong MGM_VideoDecoder_GetPosition(MGM_VideoDecoder* decoder) { - assert(video != nullptr); + assert(decoder != nullptr); + return decoder->GetPosition(); } -void MGM_Video_Stop(MGM_Video* video) +void MGM_VideoDecoder_SetLooped(MGM_VideoDecoder* decoder, mgbool looped) { - assert(video != nullptr); + assert(decoder != nullptr); + decoder->SetLooped(looped); } -void MGM_Video_GetFrame(MGM_Video* video, mguint& frame, MGG_Texture*& handle) +MGG_Texture* MGM_VideoDecoder_Decode(MGM_VideoDecoder* decoder) { - assert(video != nullptr); - frame = 0; - handle = nullptr; + assert(decoder != nullptr); + return decoder->Decode(); } diff --git a/native/monogame/faudio/MGA_faudio.cpp b/native/monogame/faudio/MGA_faudio.cpp index 00d3783a58f..e2b688ca45a 100644 --- a/native/monogame/faudio/MGA_faudio.cpp +++ b/native/monogame/faudio/MGA_faudio.cpp @@ -91,7 +91,7 @@ mgulong MGA_Buffer_GetDuration(MGA_Buffer* buffer) return 0; } -MGA_Voice* MGA_Voice_Create(MGA_System* system) +MGA_Voice* MGA_Voice_Create(MGA_System* system, mgint sampleRate, mgint channels) { assert(system != nullptr); auto voice = new MGA_Voice(); @@ -123,16 +123,11 @@ void MGA_Voice_SetBuffer(MGA_Voice* voice, MGA_Buffer* buffer) } } -void MGA_Voice_AppendBuffer(MGA_Voice* voice, mgbyte* buffer, mgint offset, mgint count, mgbool clear) +void MGA_Voice_AppendBuffer(MGA_Voice* voice, mgbyte* buffer, mguint size) { assert(voice != nullptr); assert(buffer != nullptr); - if (clear) - { - // Stop and remove any pending buffers first. - } - // The idea here is that for streaming cases and dynamic buffers // we internally allocate a big chunk of memory for it and // break it up into smaller buffers for submission. @@ -167,6 +162,12 @@ MGSoundState MGA_Voice_GetState(MGA_Voice* voice) return MGSoundState::Stopped; } +mgulong MGA_Voice_GetPosition(MGA_Voice* voice) +{ + assert(voice != nullptr); + return 0; +} + void MGA_Voice_SetPan(MGA_Voice* voice, mgfloat pan) { assert(voice != nullptr); diff --git a/native/monogame/include/MGM_common.h b/native/monogame/include/MGM_common.h new file mode 100644 index 00000000000..426dce66f60 --- /dev/null +++ b/native/monogame/include/MGM_common.h @@ -0,0 +1,55 @@ +// MonoGame - Copyright (C) The MonoGame Team +// This file is subject to the terms and conditions defined in +// file 'LICENSE.txt', which is part of this source code package. + +#include "mg_common.h" + +struct MGM_AudioDecoderInfo; +struct MGG_Texture; + + +/// +/// The base for all audio decoders. +/// +struct MGM_AudioDecoder +{ + virtual ~MGM_AudioDecoder() {} + virtual void Initialize(const char* filepath, MGM_AudioDecoderInfo& info) = 0; + virtual void SetPosition(mgulong timeMs) = 0; + virtual bool Decode(mgbyte*& buffer, mguint& size) = 0; +}; + + +/// +/// The base for all video decoders. +/// +struct MGM_VideoDecoder +{ + virtual ~MGM_VideoDecoder() {} + virtual void Initialize(const char* filepath, MGM_VideoDecoderInfo& info) = 0; + virtual MGM_AudioDecoder* GetAudioDecoder(MGM_AudioDecoderInfo& info) = 0; + virtual mgulong GetPosition() = 0; + virtual void SetLooped(mgbool looped) = 0; + virtual MGG_Texture* Decode() = 0; +}; + + + +// This seems like enough to detect most file formats. +#define MGM_SIGNATURE char signature[16] + + +/// +/// Helper to read first bytes of a file to get its signature. +/// +void MGM_ReadSignature(const char* filepath, MGM_SIGNATURE); + + +// These are the common decoders supported on all platforms. +// They all work via optimized software decoding. + +MGM_AudioDecoder* MGM_AudioDecoder_TryCreate_Ogg(MGM_SIGNATURE); +MGM_AudioDecoder* MGM_AudioDecoder_TryCreate_Mp3(MGM_SIGNATURE); + +MGM_VideoDecoder* MGM_VideoDecoder_TryCreate_Theora(MGM_SIGNATURE); +MGM_VideoDecoder* MGM_VideoDecoder_TryCreate_OpenH264(MGM_SIGNATURE); diff --git a/native/monogame/include/api_MGA.h b/native/monogame/include/api_MGA.h index 3870c4573b7..3592463eeb8 100644 --- a/native/monogame/include/api_MGA.h +++ b/native/monogame/include/api_MGA.h @@ -26,16 +26,17 @@ MG_EXPORT void MGA_Buffer_InitializeFormat(MGA_Buffer* buffer, mgbyte* waveHeade MG_EXPORT void MGA_Buffer_InitializePCM(MGA_Buffer* buffer, mgbyte* waveData, mgint offset, mgint length, mgint sampleBits, mgint sampleRate, mgint channels, mgint loopStart, mgint loopLength); MG_EXPORT void MGA_Buffer_InitializeXact(MGA_Buffer* buffer, mguint codec, mgbyte* waveData, mgint length, mgint sampleRate, mgint blockAlignment, mgint channels, mgint loopStart, mgint loopLength); MG_EXPORT mgulong MGA_Buffer_GetDuration(MGA_Buffer* buffer); -MG_EXPORT MGA_Voice* MGA_Voice_Create(MGA_System* system); +MG_EXPORT MGA_Voice* MGA_Voice_Create(MGA_System* system, mgint sampleRate, mgint channels); MG_EXPORT void MGA_Voice_Destroy(MGA_Voice* voice); MG_EXPORT mgint MGA_Voice_GetBufferCount(MGA_Voice* voice); MG_EXPORT void MGA_Voice_SetBuffer(MGA_Voice* voice, MGA_Buffer* buffer); -MG_EXPORT void MGA_Voice_AppendBuffer(MGA_Voice* voice, mgbyte* buffer, mgint offset, mgint count, mgbool clear); +MG_EXPORT void MGA_Voice_AppendBuffer(MGA_Voice* voice, mgbyte* buffer, mguint size); MG_EXPORT void MGA_Voice_Play(MGA_Voice* voice, mgbool looped); MG_EXPORT void MGA_Voice_Pause(MGA_Voice* voice); MG_EXPORT void MGA_Voice_Resume(MGA_Voice* voice); MG_EXPORT void MGA_Voice_Stop(MGA_Voice* voice, mgbool immediate); MG_EXPORT MGSoundState MGA_Voice_GetState(MGA_Voice* voice); +MG_EXPORT mgulong MGA_Voice_GetPosition(MGA_Voice* voice); MG_EXPORT void MGA_Voice_SetPan(MGA_Voice* voice, mgfloat pan); MG_EXPORT void MGA_Voice_SetPitch(MGA_Voice* voice, mgfloat pitch); MG_EXPORT void MGA_Voice_SetVolume(MGA_Voice* voice, mgfloat volume); diff --git a/native/monogame/include/api_MGM.h b/native/monogame/include/api_MGM.h index a35b6c294b2..43c9a674b25 100644 --- a/native/monogame/include/api_MGM.h +++ b/native/monogame/include/api_MGM.h @@ -12,27 +12,18 @@ #include "api_structs.h" -struct MGM_Song; -struct MGM_Video; +struct MGM_AudioDecoder; +struct MGM_VideoDecoder; +struct MGG_GraphicsDevice; +struct MGG_Texture; -MG_EXPORT MGM_Song* MGM_Song_Create(const char* mediaFilePath); -MG_EXPORT void MGM_Song_Destroy(MGM_Song* song); -MG_EXPORT mgulong MGM_Song_GetDuration(MGM_Song* song); -MG_EXPORT mgulong MGM_Song_GetPosition(MGM_Song* song); -MG_EXPORT mgfloat MGM_Song_GetVolume(MGM_Song* song); -MG_EXPORT void MGM_Song_SetVolume(MGM_Song* song, mgfloat volume); -MG_EXPORT void MGM_Song_Play(MGM_Song* song, mgulong startPositionMs, void (*callback)(void*), void* callbackData); -MG_EXPORT void MGM_Song_Pause(MGM_Song* song); -MG_EXPORT void MGM_Song_Resume(MGM_Song* song); -MG_EXPORT void MGM_Song_Stop(MGM_Song* song); -MG_EXPORT MGM_Video* MGM_Video_Create(const char* mediaFilePath, mgint cachedFrameNum, mgint& width, mgint& height, mgfloat& fps, mgulong& duration); -MG_EXPORT void MGM_Video_Destroy(MGM_Video* video); -MG_EXPORT MGMediaState MGM_Video_GetState(MGM_Video* video); -MG_EXPORT mgulong MGM_Video_GetPosition(MGM_Video* video); -MG_EXPORT void MGM_Video_SetVolume(MGM_Video* video, mgfloat volume); -MG_EXPORT void MGM_Video_SetLooped(MGM_Video* video, mgbool looped); -MG_EXPORT void MGM_Video_Play(MGM_Video* video); -MG_EXPORT void MGM_Video_Pause(MGM_Video* video); -MG_EXPORT void MGM_Video_Resume(MGM_Video* video); -MG_EXPORT void MGM_Video_Stop(MGM_Video* video); -MG_EXPORT void MGM_Video_GetFrame(MGM_Video* video, mguint& frame, MGG_Texture*& handle); +MG_EXPORT MGM_AudioDecoder* MGM_AudioDecoder_Create(const char* filepath, MGM_AudioDecoderInfo& info); +MG_EXPORT void MGM_AudioDecoder_Destroy(MGM_AudioDecoder* decoder); +MG_EXPORT void MGM_AudioDecoder_SetPosition(MGM_AudioDecoder* decoder, mgulong timeMS); +MG_EXPORT mgbool MGM_AudioDecoder_Decode(MGM_AudioDecoder* decoder, mgbyte*& buffer, mguint& size); +MG_EXPORT MGM_VideoDecoder* MGM_VideoDecoder_Create(MGG_GraphicsDevice* device, const char* filepath, MGM_VideoDecoderInfo& info); +MG_EXPORT void MGM_VideoDecoder_Destroy(MGM_VideoDecoder* decoder); +MG_EXPORT MGM_AudioDecoder* MGM_VideoDecoder_GetAudioDecoder(MGM_VideoDecoder* decoder, MGM_AudioDecoderInfo& info); +MG_EXPORT mgulong MGM_VideoDecoder_GetPosition(MGM_VideoDecoder* decoder); +MG_EXPORT void MGM_VideoDecoder_SetLooped(MGM_VideoDecoder* decoder, mgbool looped); +MG_EXPORT MGG_Texture* MGM_VideoDecoder_Decode(MGM_VideoDecoder* decoder); diff --git a/native/monogame/include/api_enums.h b/native/monogame/include/api_enums.h index a878ece5a2d..fc091fda7fc 100644 --- a/native/monogame/include/api_enums.h +++ b/native/monogame/include/api_enums.h @@ -244,13 +244,6 @@ enum class MGVertexElementFormat : mgint HalfVector4 = 11, }; -enum class MGMediaState : mgint -{ - Stopped = 0, - Playing = 1, - Paused = 2, -}; - enum class MGGameRunBehavior : mgint { Asynchronous = 0, diff --git a/native/monogame/include/api_structs.h b/native/monogame/include/api_structs.h index 6745b5ce01c..94e9ced54db 100644 --- a/native/monogame/include/api_structs.h +++ b/native/monogame/include/api_structs.h @@ -155,6 +155,21 @@ struct MGG_InputElement mguint InstanceDataStepRate; }; +struct MGM_AudioDecoderInfo +{ + mgint samplerate; + mgint channels; + mgulong duration; +}; + +struct MGM_VideoDecoderInfo +{ + mgint width; + mgint height; + mgfloat fps; + mgulong duration; +}; + struct MGP_KeyEvent { void* Window;