Skip to content

Commit

Permalink
Amend AudioSampleBufferConverter to support various encoder options
Browse files Browse the repository at this point in the history 
https://bugs.webkit.org/show_bug.cgi?id=284355
rdar://141204354

Reviewed by NOBODY (OOPS!).

Add support for passing various encoder options to the AudioSampleBufferConverter's AudioConverter.

The size of the compressed packet is determined by setting the AudioStreamBasicDescription's mFramesPerPacket.

Tests to follow as part of the WebCodec's WPT. No change of behaviours at this stage.

* Source/WebCore/platform/audio/cocoa/AudioSampleBufferConverter.h:
(WebCore::AudioSampleBufferConverter::preSkip const): Add method to retrieve the preSkip value after creating the AudioConverter.
Will be used to create the Opus's codec description.
* Source/WebCore/platform/audio/cocoa/AudioSampleBufferConverter.mm:
(WebCore::AudioSampleBufferConverter::initAudioConverterForSourceFormatDescription):
  • Loading branch information
@jyavenard
jyavenard committed Dec 11, 2024
1 parent 9ddd721 commit e51276f
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 8 deletions.
21 changes: 20 additions & 1 deletion Source/WebCore/platform/audio/cocoa/AudioSampleBufferConverter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

#if USE(AVFOUNDATION)

#include "BitrateMode.h"
#include <CoreMedia/CoreMedia.h>
#include <wtf/Forward.h>
#include <wtf/ThreadSafeWeakPtr.h>
Expand All @@ -41,12 +42,28 @@ class WebAudioBufferList;

class AudioSampleBufferConverter : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr<AudioSampleBufferConverter> {
public:

#if ENABLE(WEB_CODECS)
using BitrateMode = BitrateMode;
#else
enum class BitrateMode {
Constant,
Variable
};
#endif

struct Options {
AudioFormatID format { kAudioFormatMPEG4AAC };
std::optional<AudioStreamBasicDescription> description { };
std::optional<unsigned> outputBitRate { };
bool generateTimestamp { true };
std::optional<unsigned> preSkip { }; // If not set, let AudioConverter use the default.
std::optional<unsigned> preSkip { };
std::optional<BitrateMode> bitrateMode { };
std::optional<unsigned> packetSize { };
std::optional<unsigned> complexity { };
std::optional<unsigned> packetlossperc { };
std::optional<bool> useinbandfec { };
std::optional<bool> usedtx { };
};
static RefPtr<AudioSampleBufferConverter> create(CMBufferQueueTriggerCallback, void* callbackObject, const Options&);
~AudioSampleBufferConverter();
Expand All @@ -60,6 +77,7 @@ class AudioSampleBufferConverter : public ThreadSafeRefCountedAndCanMakeThreadSa
RetainPtr<CMSampleBufferRef> takeOutputSampleBuffer();

unsigned bitRate() const;
unsigned preSkip() const { return m_preSkip; }

private:
AudioSampleBufferConverter(const Options&);
Expand Down Expand Up @@ -109,6 +127,7 @@ class AudioSampleBufferConverter : public ThreadSafeRefCountedAndCanMakeThreadSa
const AudioFormatID m_outputCodecType;
const Options m_options;
std::atomic<unsigned> m_defaultBitRate { 0 };
std::atomic<unsigned> m_preSkip { 0 };
};

}
Expand Down
45 changes: 38 additions & 7 deletions Source/WebCore/platform/audio/cocoa/AudioSampleBufferConverter.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,21 @@
m_destinationFormat.mChannelsPerFrame = m_sourceFormat.mChannelsPerFrame;
m_destinationFormat.mSampleRate = m_sourceFormat.mSampleRate;
}
if (outputFormatID == kAudioFormatOpus)
m_destinationFormat.mSampleRate = 48000;

UInt32 size = sizeof(m_destinationFormat);
if (auto error = PAL::AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, &size, &m_destinationFormat)) {
if (auto error = [&](auto& destinationFormat) {
auto originalDestinationFormat = destinationFormat;
UInt32 size = sizeof(destinationFormat);
auto result = PAL::AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, &size, &destinationFormat);
if (result == kAudioCodecUnsupportedFormatError) {
destinationFormat.mSampleRate = 48000;
result = PAL::AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, &size, &destinationFormat);
if (originalDestinationFormat.mFramesPerPacket) {
// Adjust mFramesPerPacket to match new sampling rate
destinationFormat.mFramesPerPacket = originalDestinationFormat.mFramesPerPacket / originalDestinationFormat.mSampleRate * destinationFormat.mSampleRate;
}
}
return result;
}(m_destinationFormat)) {
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter AudioFormatGetProperty failed with %d", static_cast<int>(error));
return error;
}
Expand Down Expand Up @@ -209,7 +219,7 @@
}
}

size = sizeof(m_sourceFormat);
UInt32 size = sizeof(m_sourceFormat);
if (auto error = PAL::AudioConverterGetProperty(m_converter, kAudioConverterCurrentInputStreamDescription, &size, &m_sourceFormat)) {
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter getting kAudioConverterCurrentInputStreamDescription failed with %d", static_cast<int>(error));
return error;
Expand All @@ -231,8 +241,7 @@
}
if (shouldSetDefaultOutputBitRate) {
auto outputBitRate = defaultOutputBitRate(m_destinationFormat);
size = sizeof(outputBitRate);
if (auto error = PAL::AudioConverterSetProperty(m_converter, kAudioConverterEncodeBitRate, size, &outputBitRate))
if (auto error = PAL::AudioConverterSetProperty(m_converter, kAudioConverterEncodeBitRate, sizeof(outputBitRate), &outputBitRate))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter setting default kAudioConverterEncodeBitRate failed with %d", static_cast<int>(error));
else
m_defaultBitRate = outputBitRate;
Expand All @@ -250,7 +259,29 @@
UInt32 size = sizeof(primeInfo);
if (auto error = PAL::AudioConverterGetProperty(m_converter, kAudioConverterPrimeInfo, &size, &primeInfo))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter getting kAudioConverterPrimeInfo failed with %d", static_cast<int>(error));
else
m_preSkip = primeInfo.leadingFrames;
m_remainingPrimeDuration = PAL::CMTimeMake(primeInfo.leadingFrames, m_destinationFormat.mSampleRate);

if (m_options.bitrateMode) {
UInt32 bitrateMode = *m_options.bitrateMode == BitrateMode::Variable ? kAudioCodecBitRateControlMode_Variable : kAudioCodecBitRateControlMode_Constant;
if (auto error = PAL::AudioConverterSetProperty(m_converter, kAudioCodecPropertyBitRateControlMode, sizeof(bitrateMode), &bitrateMode))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter setting kAudioCodecPropertyBitRateControlMode failed with %d", static_cast<int>(error));
}
if (m_options.complexity) {
if (auto error = PAL::AudioConverterSetProperty(m_converter, kAudioCodecPropertyQualitySetting, sizeof(*m_options.complexity), &m_options.complexity.value()))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter setting kAudioCodecPropertyQualitySetting failed with %d", static_cast<int>(error));
}

// Only operational with Opus encoder.
if (m_options.packetlossperc) {
if (auto error = PAL::AudioConverterSetProperty(m_converter, 'plsp', sizeof(*m_options.packetlossperc), &m_options.packetlossperc.value()))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter setting packetlossperc failed with %d", static_cast<int>(error));
}
if (m_options.useinbandfec) {
if (auto error = PAL::AudioConverterSetProperty(m_converter, 'pfec', sizeof(*m_options.useinbandfec), &m_options.useinbandfec.value()))
RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter setting useinbandfec failed with %d", static_cast<int>(error));
}
}

if (!m_destinationFormat.mBytesPerPacket) {
Expand Down

0 comments on commit e51276f

Please sign in to comment.