diff --git a/mediapipe/objc/BUILD b/mediapipe/objc/BUILD index a2167760..7df6c802 100644 --- a/mediapipe/objc/BUILD +++ b/mediapipe/objc/BUILD @@ -204,6 +204,7 @@ objc_library( "//third_party/apple_frameworks:AVFoundation", "//third_party/apple_frameworks:CoreAudio", "//third_party/apple_frameworks:CoreMedia", + "@eigen_archive//:eigen3", ], ) diff --git a/mediapipe/objc/DrishtiAudioUtil.mm b/mediapipe/objc/DrishtiAudioUtil.mm index 83c8beda..e95435ac 100644 --- a/mediapipe/objc/DrishtiAudioUtil.mm +++ b/mediapipe/objc/DrishtiAudioUtil.mm @@ -16,50 +16,55 @@ #include +#include "third_party/eigen3/Eigen/Core" + namespace { +using Eigen::Index; +using Eigen::Map; +using Eigen::VectorXf; +using VectorXi16 = Eigen::Vector; + // `float` is 32-bit. static_assert(std::numeric_limits::is_iec559); -using float32_t = float; -template -float GetSample(const void* data, int index); - -template <> -float GetSample(const void* data, int index) { - return reinterpret_cast(data)[index]; +// Reads an array of `size` elements of type `float` at `samples` and writes it into `target`, +// which is an Eigen expression compatible with a `VectorXf` of size `size`. +template +void CopyBufferToFloatVector(const float* samples, CMItemCount size, OutputVector target) { + target = Map(samples, static_cast(size)); }; -template <> -float GetSample(const void* data, int index) { +// Reads an array of `size` elements of type `SInt16` at `samples` and writes it into `target`, +// which is an Eigen expression compatible with a `VectorXf` of size `size`. +template +void CopyBufferToFloatVector(const SInt16* samples, CMItemCount size, OutputVector target) { // Convert to the [-1, 1] range. - return static_cast(reinterpret_cast(data)[index]) / - static_cast(std::numeric_limits::max()); + constexpr float kRangeMax = static_cast(std::numeric_limits::max()); + target = Map(samples, static_cast(size)).cast() / kRangeMax; }; template -std::unique_ptr MakeMatrix(const AudioBuffer* buffers, int channels, +std::unique_ptr MakeMatrix(const AudioBuffer* buffers, CMItemCount channels, CMItemCount frames, bool interleaved) { // Create the matrix and fill it accordingly. Its dimensions are `channels x frames`. auto matrix = std::make_unique(channels, frames); - // Split the case of interleaved and non-interleaved samples (see + // Split the cases of interleaved and non-interleaved samples (see // https://developer.apple.com/documentation/coremedia/1489723-cmsamplebuffercreate#discussion) // - however, the resulting operations coincide when `channels == 1`. if (interleaved) { // A single buffer contains interleaved samples for all the channels {L, R, L, R, L, R, ...}. - const void* samples = buffers[0].mData; - for (int channel = 0; channel < channels; ++channel) { - for (int frame = 0; frame < frames; ++frame) { - (*matrix)(channel, frame) = GetSample(samples, channels * frame + channel); - } - } + // This corresponds to Eigen's default column-major matrix layout. + const SampleDataType* samples = reinterpret_cast(buffers[0].mData); + CopyBufferToFloatVector(/*samples=*/samples, /*size=*/channels * frames, + /*target=*/matrix->reshaped()); } else { // Non-interleaved audio: each channel's samples are stored in a separate buffer: // {{L, L, L, L, ...}, {R, R, R, R, ...}}. - for (int channel = 0; channel < channels; ++channel) { - const void* samples = buffers[channel].mData; - for (int frame = 0; frame < frames; ++frame) { - (*matrix)(channel, frame) = GetSample(samples, frame); - } + for (CMItemCount channel = 0; channel < channels; ++channel) { + const SampleDataType* samples = + reinterpret_cast(buffers[channel].mData); + CopyBufferToFloatVector(/*samples=*/samples, /*size=*/frames, + /*target=*/matrix->row(static_cast(channel))); } } return matrix; @@ -89,13 +94,12 @@ absl::StatusOr> MediaPipeConvertAudioBufferLi if ((streamHeader->mFormatFlags & kAudioFormatFlagIsFloat) && streamHeader->mBitsPerChannel == 32) { - return MakeMatrix(audioBufferList->mBuffers, numChannels, numFrames, - isAudioInterleaved); + return MakeMatrix(audioBufferList->mBuffers, numChannels, numFrames, isAudioInterleaved); } if ((streamHeader->mFormatFlags & kAudioFormatFlagIsSignedInteger) && streamHeader->mBitsPerChannel == 16) { return MakeMatrix(audioBufferList->mBuffers, numChannels, numFrames, isAudioInterleaved); } - return absl::InternalError("Incompatible audio sample storage format"); + return absl::InternalError("Unsupported audio sample storage format"); }