VideoToolBox decoder/encoder for MacOS & iOS (#19)

* VideoDecoderFactory wrapper

* wip

* fix compilation

* add notes

* upgrade webrtc version

.cargo/config

@@ -1,2 +1,8 @@
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-C", "target-feature=+crt-static"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-args=-ObjC"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-args=-ObjC"]

webrtc-sys/build.rs

@@ -8,7 +8,7 @@ use std::path;
 use std::process::Command;
 use tar::Archive;
 
-const WEBRTC_TAG: &str = "m104.5112.06";
+const WEBRTC_TAG: &str = "m104.5112.07";
 
 fn download_prebuilt(
     target_os: &str,
@@ -156,6 +156,8 @@ fn main() {
     builder.file("src/rtp_transceiver.cpp");
     builder.file("src/rtc_error.cpp");
     builder.file("src/webrtc.cpp");
+    builder.file("src/video_encoder_factory.cpp");
+    builder.file("src/video_decoder_factory.cpp");
 
     for include in includes {
         builder.include(include);
@@ -232,6 +234,8 @@ fn main() {
                 }
             }
 
+            builder.file("src/objc_video_factory.mm");
+
             builder
                 .flag("-stdlib=libc++")
                 .flag("-std=c++17")

webrtc-sys/include/livekit/objc_video_factory.h

@@ -0,0 +1,15 @@
+#ifndef OBJC_VIDEO_FACTORY_H
+#define OBJC_VIDEO_FACTORY_H
+
+#include <memory>
+#include "api/video_codecs/video_decoder_factory.h"
+#include "api/video_codecs/video_encoder_factory.h"
+
+namespace livekit {
+
+std::unique_ptr<webrtc::VideoEncoderFactory> CreateObjCVideoEncoderFactory();
+std::unique_ptr<webrtc::VideoDecoderFactory> CreateObjCVideoDecoderFactory();
+
+}  // namespace livekit
+
+#endif // OBJC_VIDEO_FACTORY_H

webrtc-sys/include/livekit/video_decoder_factory.h

@@ -0,0 +1,22 @@
+#ifndef VIDEO_DECODER_FACTORY_H
+#define VIDEO_DECODER_FACTORY_H
+
+#include "api/video_codecs/video_decoder_factory.h"
+#include "api/video_codecs/video_decoder.h"
+
+namespace livekit {
+class VideoDecoderFactory : public webrtc::VideoDecoderFactory {
+ public:
+  VideoDecoderFactory();
+
+  std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
+
+  std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
+      const webrtc::SdpVideoFormat& format) override;
+
+ private:
+  std::vector<std::unique_ptr<webrtc::VideoDecoderFactory>> factories_;
+};
+}  // namespace livekit
+
+#endif  // VIDEO_DECODER_FACTORY_H

webrtc-sys/include/livekit/video_encoder_factory.h

@@ -0,0 +1,22 @@
+#ifndef VIDEO_ENCODER_FACTORY_H
+#define VIDEO_ENCODER_FACTORY_H
+
+#include "api/video_codecs/video_encoder_factory.h"
+#include "api/video_codecs/video_encoder.h"
+
+namespace livekit {
+class VideoEncoderFactory : public webrtc::VideoEncoderFactory {
+ public:
+  VideoEncoderFactory();
+
+  std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
+
+  std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
+      const webrtc::SdpVideoFormat& format) override;
+
+ private:
+  std::vector<std::unique_ptr<webrtc::VideoEncoderFactory>> factories_;
+};
+}  // namespace livekit
+
+#endif  // VIDEO_ENCODER_FACTORY_H

webrtc-sys/src/objc_video_factory.mm

@@ -0,0 +1,21 @@
+#include "livekit/objc_video_factory.h"
+
+#import <sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h>
+#import <sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h>
+#include "sdk/objc/native/api/video_decoder_factory.h"
+#include "sdk/objc/native/api/video_encoder_factory.h"
+
+namespace livekit {
+
+std::unique_ptr<webrtc::VideoEncoderFactory> CreateObjCVideoEncoderFactory() {
+  // TODO(theomonnom): Simulcast?
+  return webrtc::ObjCToNativeVideoEncoderFactory(
+      [[RTCVideoEncoderFactoryH264 alloc] init]);
+}
+
+std::unique_ptr<webrtc::VideoDecoderFactory> CreateObjCVideoDecoderFactory() {
+  return webrtc::ObjCToNativeVideoDecoderFactory(
+      [[RTCVideoDecoderFactoryH264 alloc] init]);
+}
+
+}

webrtc-sys/src/peer_connection_factory.cpp

@@ -13,6 +13,8 @@
 #include "api/video_codecs/builtin_video_decoder_factory.h"
 #include "api/video_codecs/builtin_video_encoder_factory.h"
 #include "livekit/rtc_error.h"
+#include "livekit/video_decoder_factory.h"
+#include "livekit/video_encoder_factory.h"
 #include "media/engine/webrtc_media_engine.h"
 #include "webrtc-sys/src/peer_connection_factory.rs.h"
 
@@ -36,8 +38,10 @@ PeerConnectionFactory::PeerConnectionFactory(
 
   cricket::MediaEngineDependencies media_deps;
   media_deps.task_queue_factory = dependencies.task_queue_factory.get();
-  media_deps.video_encoder_factory = webrtc::CreateBuiltinVideoEncoderFactory();
+  media_deps.video_encoder_factory =
-  media_deps.video_decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory();
+      std::move(std::make_unique<livekit::VideoEncoderFactory>());
+  media_deps.video_decoder_factory =
+      std::move(std::make_unique<livekit::VideoDecoderFactory>());
   media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
   media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
   media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();

webrtc-sys/src/video_decoder_factory.cpp

@@ -0,0 +1,46 @@
+#include "livekit/video_decoder_factory.h"
+
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "livekit/objc_video_factory.h"
+#include "media/base/media_constants.h"
+#include "rtc_base/logging.h"
+
+namespace livekit {
+
+VideoDecoderFactory::VideoDecoderFactory() {
+  factories_.push_back(webrtc::CreateBuiltinVideoDecoderFactory());
+
+#ifdef __APPLE__
+  factories_.push_back(livekit::CreateObjCVideoDecoderFactory());
+#endif
+
+  // TODO(theomonnom): Add other HW decoders here
+}
+
+std::vector<webrtc::SdpVideoFormat> VideoDecoderFactory::GetSupportedFormats()
+    const {
+  std::vector<webrtc::SdpVideoFormat> formats;
+  for (const auto& factory : factories_) {
+    auto supported_formats = factory->GetSupportedFormats();
+    formats.insert(formats.end(), supported_formats.begin(),
+                   supported_formats.end());
+  }
+  return formats;
+}
+
+std::unique_ptr<webrtc::VideoDecoder> VideoDecoderFactory::CreateVideoDecoder(
+    const webrtc::SdpVideoFormat& format) {
+  for (const auto& factory : factories_) {
+    for (const auto& supported_format : factory->GetSupportedFormats()) {
+      if (supported_format.IsSameCodec(format))
+        return factory->CreateVideoDecoder(format);
+    }
+  }
+
+  RTC_LOG(LS_ERROR) << "No VideoDecoder found for " << format.name;
+  return nullptr;
+}
+
+}  // namespace livekit

webrtc-sys/src/video_encoder_factory.cpp

@@ -0,0 +1,47 @@
+#include "livekit/video_encoder_factory.h"
+
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_encoder.h"
+#include "livekit/objc_video_factory.h"
+#include "media/base/media_constants.h"
+#include "rtc_base/logging.h"
+
+namespace livekit {
+
+VideoEncoderFactory::VideoEncoderFactory() {
+  factories_.push_back(webrtc::CreateBuiltinVideoEncoderFactory());
+
+#ifdef __APPLE__
+  factories_.push_back(livekit::CreateObjCVideoEncoderFactory());
+#endif
+
+  // TODO(theomonnom): Add other HW encoders here
+}
+
+std::vector<webrtc::SdpVideoFormat> VideoEncoderFactory::GetSupportedFormats()
+    const {
+  std::vector<webrtc::SdpVideoFormat> formats;
+  for (const auto& factory : factories_) {
+    auto supported_formats = factory->GetSupportedFormats();
+    formats.insert(formats.end(), supported_formats.begin(),
+                   supported_formats.end());
+  }
+  return formats;
+}
+
+std::unique_ptr<webrtc::VideoEncoder> VideoEncoderFactory::CreateVideoEncoder(
+    const webrtc::SdpVideoFormat& format) {
+  for (const auto& factory : factories_) {
+    for (const auto& supported_format : factory->GetSupportedFormats()) {
+      if (supported_format.IsSameCodec(format))
+        return factory->CreateVideoEncoder(format);
+    }
+  }
+
+  RTC_LOG(LS_ERROR) << "No VideoEncoder found for " << format.name;
+  return nullptr;
+}
+
+}  // namespace livekit