#import "mediapipe/objc/MediaPipeAudioUtil.h" #include #include #include #include #include #import static const float kMatrixComparisonPrecisionFloat = 1e-9; static const float kMatrixComparisonPrecisionInt16 = 1e-4; @interface MediaPipeAudioUtilTest : XCTestCase @end template class AudioBufferListWrapper { public: AudioBufferListWrapper(int num_frames, int num_channels, bool interleaved) : num_frames_(num_frames), num_channels_(num_channels), interleaved_(interleaved) { int num_buffers = interleaved_ ? 1 : num_channels_; int channels_per_buffer = interleaved_ ? num_channels_ : 1; int buffer_size_samples = num_frames_ * channels_per_buffer; int buffer_size_bytes = buffer_size_samples * static_cast(BytesPerSample()); buffer_list_.reset(reinterpret_cast( calloc(1, offsetof(AudioBufferList, mBuffers) + (sizeof(AudioBuffer) * num_buffers)))); // Var. length array. assert(buffer_list_.get() != nullptr); buffer_list_->mNumberBuffers = static_cast(num_buffers); for (int buffer_index = 0; buffer_index < num_buffers; ++buffer_index) { AudioBuffer& buffer = GetBuffer(buffer_index); auto buffer_data = std::make_unique(buffer_size_samples); assert(buffer_data != nullptr); buffer.mData = static_cast(buffer_data.get()); buffer.mDataByteSize = buffer_size_bytes; buffer.mNumberChannels = channels_per_buffer; buffers_.push_back(std::move(buffer_data)); } } UInt32 BytesPerSample() const { return static_cast(sizeof(DataType)); } UInt32 BytesPerPacket() const { return static_cast(BytesPerSample() * num_frames_ * num_channels_); } AudioBufferList* GetBufferList() { return buffer_list_.get(); }; const AudioBufferList* GetBufferList() const { return buffer_list_.get(); }; AudioBuffer& GetBuffer(int index) { return GetBufferList()->mBuffers[index]; } DataType* GetBufferData(int index) { return reinterpret_cast(GetBuffer(index).mData); } DataType& At(int channel, int frame) { assert(frame >= 0 && frame < num_frames_); assert(channel >= 0 && channel < num_channels_); if (interleaved_) { // [[L, R, L, R, ...]] return GetBufferData(0)[frame * num_channels_ + channel]; } else { // [[L, L, ...], [R, R, ...]] return GetBufferData(channel)[frame]; } } DataType ToDataType(float value) const; void InitFromMatrix(const mediapipe::Matrix& matrix) { assert(matrix.rows() == num_channels_); assert(matrix.cols() == num_frames_); for (int channel = 0; channel < num_channels_; ++channel) { for (int frame = 0; frame < num_frames_; ++frame) { this->At(channel, frame) = ToDataType(matrix(channel, frame)); ; } } } private: int num_frames_; int num_channels_; bool interleaved_; std::unique_ptr buffer_list_; std::vector> buffers_; }; template <> float AudioBufferListWrapper::ToDataType(float value) const { return value; } template <> int16_t AudioBufferListWrapper::ToDataType(float value) const { return static_cast(value * std::numeric_limits::max()); } @implementation MediaPipeAudioUtilTest - (void)testBufferListToMatrixStereoNonInterleavedFloat { constexpr int kChannels = 2; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/false); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionFloat)); } - (void)testBufferListToMatrixStereoInterleavedFloat { constexpr int kChannels = 2; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/true); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionFloat)); } - (void)testBufferListToMatrixMonoNonInterleavedFloat { constexpr int kChannels = 1; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/false); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionFloat)); } - (void)testBufferListToMatrixMonoInterleavedFloat { constexpr int kChannels = 1; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/true); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionFloat)); } - (void)testBufferListToMatrixStereoNonInterleavedInt16 { constexpr int kChannels = 2; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/false); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionInt16)); } - (void)testBufferListToMatrixStereoInterleavedInt16 { constexpr int kChannels = 2; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/true); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionInt16)); } - (void)testBufferListToMatrixMonoNonInterleavedInt16 { constexpr int kChannels = 1; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/false); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionInt16)); } - (void)testBufferListToMatrixMonoInterleavedInt16 { constexpr int kChannels = 1; constexpr int kFrames = 5; mediapipe::Matrix inputMatrix(kChannels, kFrames); inputMatrix << 0, 0.1, 0.2, 0.3, 0.4; AudioBufferListWrapper bufferList(/*num_frames=*/kFrames, /*num_channels=*/kChannels, /*interleaved=*/true); bufferList.InitFromMatrix(inputMatrix); static const AudioStreamBasicDescription kStreamDescription = { .mSampleRate = 44100, .mFormatID = kAudioFormatLinearPCM, .mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, .mBytesPerPacket = bufferList.BytesPerPacket(), .mFramesPerPacket = kFrames, .mBytesPerFrame = bufferList.BytesPerSample() * kChannels, .mChannelsPerFrame = kChannels, .mBitsPerChannel = bufferList.BytesPerSample() * 8, }; absl::StatusOr> matrix = MediaPipeConvertAudioBufferListToAudioMatrix(bufferList.GetBufferList(), &kStreamDescription, static_cast(kFrames)); if (!matrix.ok()) { XCTFail(@"Unable to convert a sample buffer list to a matrix: %s", matrix.status().ToString().c_str()); } XCTAssertTrue(inputMatrix.isApprox(**matrix, kMatrixComparisonPrecisionInt16)); } @end