Project import generated by Copybara.
PiperOrigin-RevId: 263889205
This commit is contained in:
MediaPipe Team
@@ -0,0 +1,252 @@
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
cc_library(
|
||||
name = "CFHolder",
|
||||
# Header is excluded on non-ios so you can still build :all.
|
||||
hdrs = select({
|
||||
"//mediapipe:apple": ["CFHolder.h"],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "util",
|
||||
srcs = ["util.cc"],
|
||||
hdrs = ["util.h"],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":CFHolder",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "Weakify",
|
||||
hdrs = ["Weakify.h"],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
)
|
||||
|
||||
MEDIAPIPE_IOS_SRCS = [
|
||||
"MPPGraph.mm",
|
||||
"MPPTimestampConverter.mm",
|
||||
"NSError+util_status.mm",
|
||||
]
|
||||
|
||||
MEDIAPIPE_IOS_HDRS = [
|
||||
"MPPGraph.h",
|
||||
"MPPTimestampConverter.h",
|
||||
"NSError+util_status.h",
|
||||
]
|
||||
|
||||
MEDIAPIPE_IOS_CC_DEPS = [
|
||||
":CFHolder",
|
||||
":util",
|
||||
"//mediapipe/framework/port:map_util",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:source_location",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/framework/port:threadpool",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:mediapipe_profiling",
|
||||
"//mediapipe/gpu:MPPGraphGPUData",
|
||||
"//mediapipe/gpu:pixel_buffer_pool_util",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gpu_shared_data_internal",
|
||||
"//mediapipe/gpu:graph_support",
|
||||
# Other deps
|
||||
"//mediapipe/util:cpu_util",
|
||||
"@com_google_absl//absl/base:core_headers",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/synchronization",
|
||||
]
|
||||
|
||||
objc_library(
|
||||
name = "mediapipe_framework_ios",
|
||||
srcs = MEDIAPIPE_IOS_SRCS,
|
||||
hdrs = MEDIAPIPE_IOS_HDRS,
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
sdk_frameworks = [
|
||||
# Needed for OpenCV.
|
||||
"Accelerate",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = MEDIAPIPE_IOS_CC_DEPS + [
|
||||
# These are objc_library deps.
|
||||
"@google_toolbox_for_mac//:GTM_Defines",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "mediapipe_input_sources_ios",
|
||||
srcs = [
|
||||
"MPPCameraInputSource.m",
|
||||
"MPPDisplayLinkWeakTarget.m",
|
||||
"MPPInputSource.m",
|
||||
"MPPPlayerInputSource.m",
|
||||
],
|
||||
hdrs = [
|
||||
"MPPCameraInputSource.h",
|
||||
"MPPDisplayLinkWeakTarget.h",
|
||||
"MPPInputSource.h",
|
||||
"MPPPlayerInputSource.h",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "mediapipe_gl_view_renderer",
|
||||
srcs = [
|
||||
"MPPGLViewRenderer.mm",
|
||||
],
|
||||
hdrs = [
|
||||
"MPPGLViewRenderer.h",
|
||||
],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":mediapipe_framework_ios",
|
||||
"//mediapipe/gpu:gl_calculator_helper_ios",
|
||||
"//mediapipe/gpu:gl_quad_renderer",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "mediapipe_layer_renderer",
|
||||
srcs = [
|
||||
"MPPLayerRenderer.m",
|
||||
],
|
||||
hdrs = [
|
||||
"MPPLayerRenderer.h",
|
||||
],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":mediapipe_framework_ios",
|
||||
":mediapipe_gl_view_renderer",
|
||||
"//mediapipe/gpu:gl_calculator_helper_ios",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "CGImageRefUtils",
|
||||
srcs = [
|
||||
"CGImageRefUtils.mm",
|
||||
],
|
||||
hdrs = [
|
||||
"CGImageRefUtils.h",
|
||||
],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
sdk_frameworks = [
|
||||
"CoreVideo",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":mediapipe_framework_ios",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "MPPGraphTestBase",
|
||||
testonly = 1,
|
||||
srcs = [
|
||||
"MPPGraphTestBase.mm",
|
||||
],
|
||||
hdrs = [
|
||||
"MPPGraphTestBase.h",
|
||||
],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
sdk_frameworks = [
|
||||
"Accelerate",
|
||||
"AVFoundation",
|
||||
"CoreVideo",
|
||||
"CoreGraphics",
|
||||
"CoreMedia",
|
||||
"GLKit",
|
||||
"OpenGLES",
|
||||
"QuartzCore",
|
||||
"UIKit",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":CGImageRefUtils",
|
||||
":Weakify",
|
||||
":mediapipe_framework_ios",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "mediapipe_framework_ios_testLib",
|
||||
testonly = 1,
|
||||
srcs = [
|
||||
"CFHolderTests.mm",
|
||||
"MPPGraphTests.mm",
|
||||
],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
],
|
||||
data = [
|
||||
"testdata/googlelogo_color_272x92dp.png",
|
||||
],
|
||||
sdk_frameworks = [
|
||||
"Accelerate",
|
||||
"AVFoundation",
|
||||
"CoreVideo",
|
||||
"CoreGraphics",
|
||||
"CoreMedia",
|
||||
"GLKit",
|
||||
"QuartzCore",
|
||||
"UIKit",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
deps = [
|
||||
":CGImageRefUtils",
|
||||
":MPPGraphTestBase",
|
||||
":Weakify",
|
||||
":mediapipe_framework_ios",
|
||||
"//mediapipe/calculators/core:pass_through_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_binary_graph")
|
||||
|
||||
[
|
||||
mediapipe_binary_graph(
|
||||
name = graph.split("/")[-1].rsplit(".", 1)[0] + "_graph",
|
||||
graph = graph,
|
||||
output_name = "%s.binarypb" % graph.split("/")[-1].rsplit(".", 1)[0],
|
||||
visibility = ["//mediapipe/framework:mediapipe_internal"],
|
||||
)
|
||||
for graph in glob(["testdata/*.pbtxt"])
|
||||
]
|
||||
|
||||
exports_files(
|
||||
[
|
||||
"testdata/googlelogo_color_272x92dp.png",
|
||||
"testdata/googlelogo_color_272x92dp_luminance.png",
|
||||
],
|
||||
visibility = [
|
||||
"//mediapipe/feature_extraction/video/video_effects:__pkg__",
|
||||
"//mediapipe/gpu:__pkg__",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_OBJC_CFHOLDER_H_
|
||||
#define MEDIAPIPE_OBJC_CFHOLDER_H_
|
||||
|
||||
#import <CoreFoundation/CoreFoundation.h>
|
||||
|
||||
/// Manages ownership of a CoreFoundation type (any type that can be passed
|
||||
/// to CFRetain/CFRelease).
|
||||
template <typename T>
|
||||
class CFHolder {
|
||||
public:
|
||||
/// Default constructor gives a NULL ref.
|
||||
CFHolder() : _object(NULL) {}
|
||||
|
||||
/// Constructor with the basic ref type. Retains it.
|
||||
explicit CFHolder(T object) : _object(RetainIfNotNull(object)) {}
|
||||
|
||||
/// Copy constructor.
|
||||
CFHolder(const CFHolder& other) : _object(RetainIfNotNull(*other)) {}
|
||||
|
||||
/// Move constructor.
|
||||
CFHolder(CFHolder&& other) : _object(*other) { other._object = NULL; }
|
||||
|
||||
/// Destructor releases the held object.
|
||||
~CFHolder() { ReleaseIfNotNull(_object); }
|
||||
|
||||
/// Dereference to access the held object.
|
||||
T operator*() const { return _object; }
|
||||
|
||||
/// Assigning from another CFHolder adds a reference.
|
||||
CFHolder& operator=(const CFHolder& other) { return reset(*other); }
|
||||
|
||||
/// Move assignment does not add a reference.
|
||||
CFHolder& operator=(CFHolder&& other) {
|
||||
// C++11 allows its library implementation to assume that rvalue reference
|
||||
// arguments are not aliased. See 17.6.4.9 in the standard document.
|
||||
ReleaseIfNotNull(_object);
|
||||
_object = other._object;
|
||||
other._object = NULL;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Equality and inequality operators.
|
||||
bool operator==(const CFHolder& other) const {
|
||||
return _object == other._object;
|
||||
}
|
||||
bool operator!=(const CFHolder& other) const { return !operator==(other); }
|
||||
bool operator==(T object) const { return _object == object; }
|
||||
bool operator!=(T object) const { return !operator==(object); }
|
||||
|
||||
/// Sets the managed object.
|
||||
CFHolder& reset(T object) {
|
||||
T old = _object;
|
||||
_object = RetainIfNotNull(object);
|
||||
ReleaseIfNotNull(old);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Takes ownership of the object. Does not retain.
|
||||
CFHolder& adopt(T object) {
|
||||
ReleaseIfNotNull(_object);
|
||||
_object = object;
|
||||
return *this;
|
||||
}
|
||||
|
||||
private:
|
||||
static inline T RetainIfNotNull(T object) {
|
||||
if (object) CFRetain(object);
|
||||
return object;
|
||||
}
|
||||
|
||||
static inline void ReleaseIfNotNull(T object) {
|
||||
if (object) CFRelease(object);
|
||||
}
|
||||
T _object;
|
||||
};
|
||||
|
||||
/// Using these functions allows template argument deduction (i.e. you do not
|
||||
/// need to specify the type of object the holder holds, it is inferred from
|
||||
/// the argument.
|
||||
template <typename T>
|
||||
CFHolder<T>* NewCFHolder(T object) {
|
||||
return new CFHolder<T>(object);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
CFHolder<T> MakeCFHolder(T object) {
|
||||
return CFHolder<T>(object);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
CFHolder<T> MakeCFHolderAdopting(T object) {
|
||||
return CFHolder<T>().adopt(object);
|
||||
}
|
||||
|
||||
#endif // MEDIAPIPE_OBJC_CFHOLDER_H_
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <XCTest/XCTest.h>
|
||||
|
||||
#import "mediapipe/objc/CFHolder.h"
|
||||
|
||||
#include <utility>
|
||||
|
||||
@interface CFHolderTests : XCTestCase {
|
||||
UInt8 _bytes[4];
|
||||
CFDataRef _data;
|
||||
}
|
||||
@end
|
||||
|
||||
@implementation CFHolderTests
|
||||
|
||||
- (void)setUp {
|
||||
_data = CFDataCreate(NULL, _bytes, sizeof(_bytes));
|
||||
}
|
||||
|
||||
- (void)tearDown {
|
||||
CFRelease(_data);
|
||||
}
|
||||
|
||||
- (void)testCreateAndDestroy {
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
{
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
|
||||
- (void)testDereference {
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(*holder, _data);
|
||||
}
|
||||
|
||||
- (void)testCopy {
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
{
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
{
|
||||
CFHolder<CFDataRef> holder2(holder);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 3);
|
||||
{
|
||||
CFHolder<CFDataRef> holder3 = holder;
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 4);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 3);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
|
||||
- (void)testOverwriteWithNull {
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
{
|
||||
// Copy assignment.
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
CFHolder<CFDataRef> holder2;
|
||||
holder = holder2;
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
{
|
||||
// Move assignment.
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
holder = CFHolder<CFDataRef>();
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)testOverwriteWithOther {
|
||||
CFDataRef data2 = CFDataCreate(NULL, _bytes, sizeof(_bytes));
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
XCTAssertEqual(CFGetRetainCount(data2), 1);
|
||||
{
|
||||
// Copy assignment.
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
CFHolder<CFDataRef> holder2(data2);
|
||||
XCTAssertEqual(CFGetRetainCount(data2), 2);
|
||||
holder = holder2;
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
XCTAssertEqual(CFGetRetainCount(data2), 3);
|
||||
}
|
||||
{
|
||||
// Move assignment.
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
holder = CFHolder<CFDataRef>(data2);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
XCTAssertEqual(CFGetRetainCount(data2), 2);
|
||||
}
|
||||
CFRelease(data2);
|
||||
}
|
||||
|
||||
- (void)testCompare {
|
||||
CFDataRef data2 = CFDataCreate(NULL, _bytes, sizeof(_bytes));
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
CFHolder<CFDataRef> holdersame(_data);
|
||||
CFHolder<CFDataRef> holderother(data2);
|
||||
CFHolder<CFDataRef> empty;
|
||||
// Compare with other holder.
|
||||
XCTAssertEqual(holder, holder);
|
||||
XCTAssertEqual(holder, holdersame);
|
||||
XCTAssertNotEqual(holder, holderother);
|
||||
XCTAssertNotEqual(holder, empty);
|
||||
// Compare with held type.
|
||||
XCTAssertEqual(holder, _data);
|
||||
XCTAssertNotEqual(holder, data2);
|
||||
XCTAssertNotEqual(holder, nil);
|
||||
XCTAssertEqual(empty, nil);
|
||||
XCTAssertNotEqual(empty, _data);
|
||||
|
||||
CFRelease(data2);
|
||||
}
|
||||
|
||||
- (void)testReset {
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
{
|
||||
CFHolder<CFDataRef> holder;
|
||||
holder.reset(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
CFDataRef data2 = CFDataCreate(NULL, _bytes, sizeof(_bytes));
|
||||
holder.reset(data2);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
XCTAssertEqual(CFGetRetainCount(data2), 2);
|
||||
CFRelease(data2);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
|
||||
- (void)testAdopt {
|
||||
CFRetain(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
{
|
||||
CFHolder<CFDataRef> holder;
|
||||
holder.adopt(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
|
||||
- (void)testMove {
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
{
|
||||
CFHolder<CFDataRef> holder(_data);
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
CFHolder<CFDataRef> holder2(std::move(holder));
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
CFHolder<CFDataRef> holder3(std::move(holder2));
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 2);
|
||||
}
|
||||
XCTAssertEqual(CFGetRetainCount(_data), 1);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_OBJC_CGIMAGEREFUTILS_H_
|
||||
#define MEDIAPIPE_OBJC_CGIMAGEREFUTILS_H_
|
||||
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif // __cplusplus
|
||||
|
||||
extern NSString *const kCGImageRefUtilsErrorDomain;
|
||||
|
||||
// TODO: Get rid of this library or make it a wrapper around util.h
|
||||
// versions so that it can be used in pure Objective-C code.
|
||||
|
||||
/// Creates a CGImage with a copy of the contents of the CVPixelBuffer. Returns nil on error, if
|
||||
/// the |error| argument is not nil, *error is set to an NSError describing the failure. Caller
|
||||
/// is responsible for releasing the CGImage by calling CGImageRelease().
|
||||
CGImageRef CreateCGImageFromCVPixelBuffer(CVPixelBufferRef imageBuffer, NSError **error);
|
||||
|
||||
/// Creates a CVPixelBuffer with a copy of the contents of the CGImage. Returns nil on error, if
|
||||
/// the |error| argument is not nil, *error is set to an NSError describing the failure. Caller
|
||||
/// is responsible for releasing the CVPixelBuffer by calling CVPixelBufferRelease.
|
||||
CVPixelBufferRef CreateCVPixelBufferFromCGImage(CGImageRef image, NSError **error);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif // __cplusplus
|
||||
|
||||
#endif // MEDIAPIPE_OBJC_CGIMAGEREFUTILS_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "CGImageRefUtils.h"
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "mediapipe/objc/CFHolder.h"
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
#import "mediapipe/objc/util.h"
|
||||
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
CGImageRef CreateCGImageFromCVPixelBuffer(CVPixelBufferRef imageBuffer, NSError **error) {
|
||||
CFHolder<CGImageRef> cg_image_holder;
|
||||
::mediapipe::Status status = CreateCGImageFromCVPixelBuffer(imageBuffer, &cg_image_holder);
|
||||
if (!status.ok()) {
|
||||
*error = [NSError gus_errorWithStatus:status];
|
||||
return nil;
|
||||
}
|
||||
CGImageRef cg_image = *cg_image_holder;
|
||||
CGImageRetain(cg_image);
|
||||
return cg_image;
|
||||
}
|
||||
|
||||
CVPixelBufferRef CreateCVPixelBufferFromCGImage(CGImageRef image, NSError **error) {
|
||||
CFHolder<CVPixelBufferRef> pixel_buffer_holder;
|
||||
::mediapipe::Status status = CreateCVPixelBufferFromCGImage(image, &pixel_buffer_holder);
|
||||
if (!status.ok()) {
|
||||
*error = [NSError gus_errorWithStatus:status];
|
||||
return nil;
|
||||
}
|
||||
CVPixelBufferRef pixel_buffer = *pixel_buffer_holder;
|
||||
CVPixelBufferRetain(pixel_buffer);
|
||||
return pixel_buffer;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#import "mediapipe/objc/MPPInputSource.h"
|
||||
|
||||
/// A source that obtains video frames from the camera.
|
||||
@interface MPPCameraInputSource : MPPInputSource
|
||||
|
||||
/// Whether we are allowed to use the camera.
|
||||
@property(nonatomic, getter=isAuthorized, readonly) BOOL authorized;
|
||||
|
||||
/// Session preset to use for capturing.
|
||||
@property(nonatomic) NSString *sessionPreset;
|
||||
|
||||
/// Which camera on an iOS device to use, assuming iOS device with more than one camera.
|
||||
@property(nonatomic) AVCaptureDevicePosition cameraPosition;
|
||||
|
||||
// Whether to use depth data or not
|
||||
@property(nonatomic) BOOL useDepth;
|
||||
|
||||
/// Whether to rotate video buffers with device rotation.
|
||||
@property(nonatomic) BOOL autoRotateBuffers;
|
||||
|
||||
/// The capture session.
|
||||
@property(nonatomic, readonly) AVCaptureSession *session;
|
||||
|
||||
/// The capture video preview layer.
|
||||
@property(nonatomic, readonly) AVCaptureVideoPreviewLayer *videoPreviewLayer;
|
||||
|
||||
/// The orientation of camera frame buffers.
|
||||
@property(nonatomic) AVCaptureVideoOrientation orientation;
|
||||
|
||||
/// Prompts the user to grant camera access and provides the result as a BOOL to a completion
|
||||
/// handler. Should be called after [MPPCameraInputSource init] and before
|
||||
/// [MPPCameraInputSource start]. If the user has previously granted or denied permission, this
|
||||
/// method simply returns the saved response to the permission request.
|
||||
- (void)requestCameraAccessWithCompletionHandler:(void (^_Nullable)(BOOL granted))handler;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,298 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPCameraInputSource.h"
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface MPPCameraInputSource () <AVCaptureVideoDataOutputSampleBufferDelegate,
|
||||
AVCaptureDepthDataOutputDelegate>
|
||||
@end
|
||||
|
||||
@implementation MPPCameraInputSource {
|
||||
AVCaptureSession* _session;
|
||||
AVCaptureDeviceInput* _videoDeviceInput;
|
||||
AVCaptureVideoDataOutput* _videoDataOutput;
|
||||
AVCaptureDepthDataOutput* _depthDataOutput;
|
||||
AVCaptureDevice *_currentDevice;
|
||||
|
||||
OSType _pixelFormatType;
|
||||
BOOL _autoRotateBuffers;
|
||||
BOOL _setupDone;
|
||||
BOOL _useDepth;
|
||||
BOOL _useCustomOrientation;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_cameraPosition = AVCaptureDevicePositionBack;
|
||||
_session = [[AVCaptureSession alloc] init];
|
||||
_pixelFormatType = kCVPixelFormatType_32BGRA;
|
||||
|
||||
AVAuthorizationStatus status =
|
||||
[AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
|
||||
_authorized = status == AVAuthorizationStatusAuthorized;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)setDelegate:(id<MPPInputSourceDelegate>)delegate
|
||||
queue:(dispatch_queue_t)queue {
|
||||
[super setDelegate:delegate queue:queue];
|
||||
// Note that _depthDataOutput and _videoDataOutput may not have been created yet. In that case,
|
||||
// this message to nil is ignored, and the delegate will be set later by setupCamera.
|
||||
[_videoDataOutput setSampleBufferDelegate:self queue:queue];
|
||||
[_depthDataOutput setDelegate:self callbackQueue:queue];
|
||||
}
|
||||
|
||||
- (void)start {
|
||||
if (!_setupDone) [self setupCamera];
|
||||
if (_autoRotateBuffers) {
|
||||
[self enableAutoRotateBufferObserver:YES];
|
||||
}
|
||||
[_session startRunning];
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
if (_autoRotateBuffers) {
|
||||
[self enableAutoRotateBufferObserver:NO];
|
||||
}
|
||||
[_session stopRunning];
|
||||
}
|
||||
|
||||
- (BOOL)isRunning {
|
||||
return _session.isRunning;
|
||||
}
|
||||
|
||||
- (void)setCameraPosition:(AVCaptureDevicePosition)cameraPosition {
|
||||
BOOL wasRunning = [self isRunning];
|
||||
if (wasRunning) {
|
||||
[self stop];
|
||||
}
|
||||
_cameraPosition = cameraPosition;
|
||||
_setupDone = NO;
|
||||
if (wasRunning) {
|
||||
[self start];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setUseDepth:(BOOL)useDepth {
|
||||
if (useDepth == _useDepth) {
|
||||
return;
|
||||
}
|
||||
|
||||
BOOL wasRunning = [self isRunning];
|
||||
if (wasRunning) {
|
||||
[self stop];
|
||||
}
|
||||
_useDepth = useDepth;
|
||||
_setupDone = NO;
|
||||
if (wasRunning) {
|
||||
[self start];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setOrientation:(AVCaptureVideoOrientation)orientation {
|
||||
if (orientation == _orientation) {
|
||||
return;
|
||||
}
|
||||
|
||||
BOOL wasRunning = [self isRunning];
|
||||
if (wasRunning) {
|
||||
[self stop];
|
||||
}
|
||||
|
||||
_orientation = orientation;
|
||||
_useCustomOrientation = YES;
|
||||
_setupDone = NO;
|
||||
if (wasRunning) {
|
||||
[self start];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setAutoRotateBuffers:(BOOL)autoRotateBuffers {
|
||||
if (autoRotateBuffers == _autoRotateBuffers) {
|
||||
return; // State has not changed.
|
||||
}
|
||||
_autoRotateBuffers = autoRotateBuffers;
|
||||
if ([self isRunning]) {
|
||||
// Enable or disable observer this settings changes while this input source is running.
|
||||
[self enableAutoRotateBufferObserver:_autoRotateBuffers];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)enableAutoRotateBufferObserver:(BOOL)enable {
|
||||
if (enable) {
|
||||
[[NSNotificationCenter defaultCenter] addObserver:self
|
||||
selector:@selector(deviceOrientationChanged)
|
||||
name:UIDeviceOrientationDidChangeNotification
|
||||
object:nil];
|
||||
// Trigger a device orientation change instead of waiting for the first change.
|
||||
[self deviceOrientationChanged];
|
||||
} else {
|
||||
[[NSNotificationCenter defaultCenter] removeObserver:self
|
||||
name:UIDeviceOrientationDidChangeNotification
|
||||
object:nil];
|
||||
}
|
||||
}
|
||||
|
||||
- (OSType)pixelFormatType {
|
||||
return _pixelFormatType;
|
||||
}
|
||||
|
||||
- (void)setPixelFormatType:(OSType)pixelFormatType {
|
||||
_pixelFormatType = pixelFormatType;
|
||||
if ([self isRunning]) {
|
||||
_videoDataOutput.videoSettings = @{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(_pixelFormatType)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - Camera-specific methods
|
||||
|
||||
- (NSString*)sessionPreset {
|
||||
return _session.sessionPreset;
|
||||
}
|
||||
|
||||
- (void)setSessionPreset:(NSString*)sessionPreset {
|
||||
_session.sessionPreset = sessionPreset;
|
||||
}
|
||||
|
||||
- (void)setupCamera {
|
||||
NSError* error = nil;
|
||||
|
||||
if (_videoDeviceInput) {
|
||||
[_session removeInput:_videoDeviceInput];
|
||||
}
|
||||
|
||||
AVCaptureDeviceDiscoverySession* deviceDiscoverySession = [AVCaptureDeviceDiscoverySession
|
||||
discoverySessionWithDeviceTypes:@[
|
||||
_cameraPosition == AVCaptureDevicePositionFront && _useDepth ?
|
||||
AVCaptureDeviceTypeBuiltInTrueDepthCamera :
|
||||
AVCaptureDeviceTypeBuiltInWideAngleCamera]
|
||||
mediaType:AVMediaTypeVideo
|
||||
position:_cameraPosition];
|
||||
AVCaptureDevice* videoDevice =
|
||||
[deviceDiscoverySession devices]
|
||||
? [deviceDiscoverySession devices].firstObject
|
||||
: [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
|
||||
_videoDeviceInput = [AVCaptureDeviceInput deviceInputWithDevice:videoDevice error:&error];
|
||||
if (error) {
|
||||
NSLog(@"%@", error);
|
||||
return;
|
||||
}
|
||||
[_session addInput:_videoDeviceInput];
|
||||
|
||||
if (!_videoDataOutput) {
|
||||
_videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
|
||||
[_session addOutput:_videoDataOutput];
|
||||
|
||||
// Set this when we have a handler.
|
||||
if (self.delegateQueue)
|
||||
[_videoDataOutput setSampleBufferDelegate:self queue:self.delegateQueue];
|
||||
_videoDataOutput.alwaysDiscardsLateVideoFrames = YES;
|
||||
|
||||
// Only a few pixel formats are available for capture output:
|
||||
// kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
|
||||
// kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
|
||||
// kCVPixelFormatType_32BGRA.
|
||||
_videoDataOutput.videoSettings = @{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(_pixelFormatType)
|
||||
};
|
||||
}
|
||||
|
||||
// Remove Old Depth Depth
|
||||
if (_depthDataOutput) {
|
||||
[_session removeOutput:_depthDataOutput];
|
||||
}
|
||||
|
||||
if (_useDepth) {
|
||||
// Add Depth Output
|
||||
_depthDataOutput = [[AVCaptureDepthDataOutput alloc] init];
|
||||
_depthDataOutput.alwaysDiscardsLateDepthData = YES;
|
||||
if ([_session canAddOutput:_depthDataOutput]) {
|
||||
[_session addOutput:_depthDataOutput];
|
||||
|
||||
AVCaptureConnection* connection =
|
||||
[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData];
|
||||
|
||||
if (connection != nil)
|
||||
connection.enabled = true;
|
||||
|
||||
// Set this when we have a handler.
|
||||
if (self.delegateQueue) {
|
||||
[_depthDataOutput setDelegate:self callbackQueue:self.delegateQueue];
|
||||
}
|
||||
}
|
||||
else
|
||||
_depthDataOutput = nil;
|
||||
}
|
||||
|
||||
if (_useCustomOrientation) {
|
||||
AVCaptureConnection* connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
||||
connection.videoOrientation = _orientation;
|
||||
}
|
||||
|
||||
_setupDone = YES;
|
||||
}
|
||||
|
||||
- (void)requestCameraAccessWithCompletionHandler:(void (^)(BOOL))handler {
|
||||
[AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo
|
||||
completionHandler:^(BOOL granted) {
|
||||
_authorized = granted;
|
||||
if (handler) {
|
||||
handler(granted);
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
#pragma mark - AVCaptureVideoDataOutputSampleBufferDelegate methods
|
||||
|
||||
// Receives frames from the camera. Invoked on self.frameHandlerQueue.
|
||||
- (void)captureOutput:(AVCaptureOutput*)captureOutput
|
||||
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection*)connection {
|
||||
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
|
||||
if ([self.delegate respondsToSelector:@selector(processVideoFrame:timestamp:fromSource:)]) {
|
||||
[self.delegate processVideoFrame:imageBuffer timestamp:timestamp fromSource:self];
|
||||
} else if ([self.delegate respondsToSelector:@selector(processVideoFrame:fromSource:)]) {
|
||||
[self.delegate processVideoFrame:imageBuffer fromSource:self];
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - AVCaptureDepthDataOutputDelegate methods
|
||||
|
||||
// Receives depth frames from the camera. Invoked on self.frameHandlerQueue.
|
||||
- (void)depthDataOutput:(AVCaptureDepthDataOutput *)output
|
||||
didOutputDepthData:(AVDepthData *)depthData
|
||||
timestamp:(CMTime)timestamp
|
||||
connection:(AVCaptureConnection *)connection {
|
||||
if (depthData.depthDataType != kCVPixelFormatType_DepthFloat32) {
|
||||
depthData = [depthData depthDataByConvertingToDepthDataType:kCVPixelFormatType_DepthFloat32];
|
||||
}
|
||||
CVPixelBufferRef depthBuffer = depthData.depthDataMap;
|
||||
[self.delegate processDepthData:depthData timestamp:timestamp fromSource:self];
|
||||
}
|
||||
|
||||
#pragma mark - NSNotificationCenter event handlers
|
||||
|
||||
- (void)deviceOrientationChanged {
|
||||
AVCaptureConnection* connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
||||
connection.videoOrientation = (AVCaptureVideoOrientation)[UIDevice currentDevice].orientation;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/// A generic target/callback holder. Useful for indirectly using DisplayLink and allowing the
|
||||
/// complete deletion of displaylink reference holders.
|
||||
@interface MPPDisplayLinkWeakTarget : NSObject
|
||||
|
||||
- (instancetype)initWithTarget:(id)target selector:(SEL)sel;
|
||||
|
||||
- (void)displayLinkCallback:(CADisplayLink *)sender;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/objc/MPPDisplayLinkWeakTarget.h"
|
||||
|
||||
@implementation MPPDisplayLinkWeakTarget {
|
||||
__weak id _target;
|
||||
SEL _selector;
|
||||
}
|
||||
|
||||
#pragma mark - Init
|
||||
|
||||
- (instancetype)initWithTarget:(id)target selector:(SEL)sel {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_target = target;
|
||||
_selector = sel;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - Public
|
||||
|
||||
- (void)displayLinkCallback:(CADisplayLink *)sender {
|
||||
void (*display)(id, SEL, CADisplayLink *) = (void *)[_target methodForSelector:_selector];
|
||||
display(_target, _selector, sender);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <GLKit/GLKit.h>
|
||||
|
||||
/// Modes of clockwise rotation for input frames.
|
||||
typedef enum {
|
||||
MediaPipeFrameRotationNone,
|
||||
MediaPipeFrameRotation90,
|
||||
MediaPipeFrameRotation180,
|
||||
MediaPipeFrameRotation270
|
||||
} MediaPipeFrameRotationMode;
|
||||
|
||||
typedef enum {
|
||||
// Scale the frame up to fit the drawing area, preserving aspect ratio; may letterbox.
|
||||
MediaPipeFrameScaleFit,
|
||||
// Scale the frame up to fill the drawing area, preserving aspect ratio; may crop.
|
||||
MediaPipeFrameScaleFillAndCrop,
|
||||
} MediaPipeFrameScaleMode;
|
||||
|
||||
/// Renders frames in a GLKView.
|
||||
@interface MPPGLViewRenderer : NSObject <GLKViewDelegate>
|
||||
|
||||
/// Rendering context for display.
|
||||
@property(nonatomic) EAGLContext *glContext;
|
||||
|
||||
/// The frame to be rendered next. This pixel buffer must be unlocked, and
|
||||
/// should not be modified after handing it to the renderer.
|
||||
@property(atomic, retain) __attribute__((NSObject)) CVPixelBufferRef nextPixelBufferToRender;
|
||||
|
||||
/// When YES, the last drawn pixel buffer is retained by this object after it is drawn in the GLView
|
||||
/// for which it is a delegate. Otherwise it is released after it has been rendered.
|
||||
/// Set this property to YES when your GLView can be redrawn with the same pixel buffer, such as
|
||||
/// during an animation.
|
||||
@property(nonatomic, assign) BOOL retainsLastPixelBuffer;
|
||||
|
||||
/// Sets which way to rotate input frames before rendering them.
|
||||
/// Default value is MediaPipeFrameRotationNone.
|
||||
/// Note that changing the transform property of a GLKView once rendering has
|
||||
/// started causes problems inside GLKView. Instead, we perform the rotation
|
||||
/// in our rendering code.
|
||||
@property(nonatomic) MediaPipeFrameRotationMode frameRotationMode;
|
||||
|
||||
/// Sets how to scale the frame within the view.
|
||||
/// Default value is MediaPipeFrameScaleScaleToFit.
|
||||
@property(nonatomic) MediaPipeFrameScaleMode frameScaleMode;
|
||||
|
||||
/// If YES, swap left and right. Useful for the front camera.
|
||||
@property(nonatomic) BOOL mirrored;
|
||||
|
||||
/// Draws a pixel buffer to its context with the specified view size.
|
||||
- (void)drawPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
width:(GLfloat)viewWidth
|
||||
height:(GLfloat)viewHeight;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPGLViewRenderer.h"
|
||||
|
||||
#include <OpenGLES/ES2/gl.h>
|
||||
#include <OpenGLES/ES2/glext.h>
|
||||
#include <algorithm>
|
||||
#include <libkern/OSAtomic.h>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/gpu/gl_quad_renderer.h"
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
#import "GTMDefines.h"
|
||||
|
||||
@implementation MPPGLViewRenderer {
|
||||
/// Used to create textures for the pixel buffers to be rendered.
|
||||
/// The use of this class allows the GPU to access the buffer's memory directly,
|
||||
/// without a memory copy.
|
||||
CVOpenGLESTextureCacheRef _textureCache;
|
||||
|
||||
/// Internal renderer.
|
||||
std::unique_ptr<mediapipe::QuadRenderer> renderer_;
|
||||
|
||||
/// Used to synchronize access to _nextPixelBufferToRender.
|
||||
OSSpinLock _bufferLock;
|
||||
volatile CVPixelBufferRef _nextPixelBufferToRender;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
|
||||
_bufferLock = OS_SPINLOCK_INIT;
|
||||
_frameRotationMode = MediaPipeFrameRotationNone;
|
||||
_frameScaleMode = MediaPipeFrameScaleFit;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
if (_textureCache) CFRelease(_textureCache);
|
||||
CVPixelBufferRelease(_nextPixelBufferToRender);
|
||||
// Fixes crash during dealloc that only happens in iOS 9. More info at b/67095363.
|
||||
if ([EAGLContext currentContext] == _glContext) {
|
||||
[EAGLContext setCurrentContext:nil];
|
||||
}
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)nextPixelBufferToRender {
|
||||
OSSpinLockLock(&_bufferLock);
|
||||
CVPixelBufferRef buffer = _nextPixelBufferToRender;
|
||||
OSSpinLockUnlock(&_bufferLock);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
- (void)setNextPixelBufferToRender:(CVPixelBufferRef)buffer {
|
||||
OSSpinLockLock(&_bufferLock);
|
||||
if (_nextPixelBufferToRender != buffer) {
|
||||
CVPixelBufferRelease(_nextPixelBufferToRender);
|
||||
_nextPixelBufferToRender = buffer;
|
||||
CVPixelBufferRetain(_nextPixelBufferToRender);
|
||||
}
|
||||
OSSpinLockUnlock(&_bufferLock);
|
||||
}
|
||||
|
||||
- (void)setupGL {
|
||||
CVReturn err;
|
||||
err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _glContext, NULL, &_textureCache);
|
||||
_GTMDevAssert(err == kCVReturnSuccess,
|
||||
@"CVOpenGLESTextureCacheCreate failed: %d", err);
|
||||
|
||||
renderer_ = absl::make_unique<mediapipe::QuadRenderer>();
|
||||
auto status = renderer_->GlSetup();
|
||||
_GTMDevAssert(status.ok(),
|
||||
@"renderer setup failed: %@", [NSError gus_errorWithStatus:status]);
|
||||
}
|
||||
|
||||
mediapipe::FrameScaleMode InternalScaleMode(MediaPipeFrameScaleMode mode) {
|
||||
switch (mode) {
|
||||
case MediaPipeFrameScaleFit:
|
||||
return mediapipe::FrameScaleMode::kFit;
|
||||
case MediaPipeFrameScaleFillAndCrop:
|
||||
return mediapipe::FrameScaleMode::kFillAndCrop;
|
||||
}
|
||||
}
|
||||
|
||||
mediapipe::FrameRotation InternalRotationMode(MediaPipeFrameRotationMode rot) {
|
||||
switch (rot) {
|
||||
case MediaPipeFrameRotationNone:
|
||||
return mediapipe::FrameRotation::kNone;
|
||||
case MediaPipeFrameRotation90:
|
||||
return mediapipe::FrameRotation::k90;
|
||||
case MediaPipeFrameRotation180:
|
||||
return mediapipe::FrameRotation::k180;
|
||||
case MediaPipeFrameRotation270:
|
||||
return mediapipe::FrameRotation::k270;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)drawPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
width:(GLfloat)viewWidth
|
||||
height:(GLfloat)viewHeight {
|
||||
if (!_textureCache) [self setupGL];
|
||||
|
||||
size_t frameWidth = CVPixelBufferGetWidth(pixelBuffer);
|
||||
size_t frameHeight = CVPixelBufferGetHeight(pixelBuffer);
|
||||
CVOpenGLESTextureRef texture = NULL;
|
||||
OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
|
||||
CVReturn error;
|
||||
if (pixelFormat == kCVPixelFormatType_OneComponent8) {
|
||||
error = CVOpenGLESTextureCacheCreateTextureFromImage(
|
||||
kCFAllocatorDefault, _textureCache, pixelBuffer, NULL,
|
||||
GL_TEXTURE_2D, GL_LUMINANCE, (GLsizei)frameWidth, (GLsizei)frameHeight,
|
||||
GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture);
|
||||
} else if (pixelFormat == kCVPixelFormatType_OneComponent32Float) {
|
||||
error = CVOpenGLESTextureCacheCreateTextureFromImage(
|
||||
kCFAllocatorDefault, _textureCache, pixelBuffer, NULL,
|
||||
GL_TEXTURE_2D, GL_LUMINANCE, (GLsizei)frameWidth, (GLsizei)frameHeight,
|
||||
GL_LUMINANCE, GL_FLOAT, 0, &texture);
|
||||
} else {
|
||||
error = CVOpenGLESTextureCacheCreateTextureFromImage(
|
||||
kCFAllocatorDefault, _textureCache, pixelBuffer, NULL,
|
||||
GL_TEXTURE_2D, GL_RGBA, (GLsizei)frameWidth, (GLsizei)frameHeight,
|
||||
GL_BGRA, GL_UNSIGNED_BYTE, 0, &texture);
|
||||
}
|
||||
_GTMDevAssert(error == kCVReturnSuccess,
|
||||
@"CVOpenGLESTextureCacheCreateTextureFromImage failed: %d", error);
|
||||
|
||||
glClear(GL_COLOR_BUFFER_BIT);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture));
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
||||
|
||||
// Note: we perform a vertical flip by swapping the top and bottom coordinates.
|
||||
// CVPixelBuffers have a top left origin and OpenGL has a bottom left origin.
|
||||
auto status = renderer_->GlRender(
|
||||
frameWidth, frameHeight, viewWidth, viewHeight,
|
||||
InternalScaleMode(_frameScaleMode),
|
||||
InternalRotationMode(_frameRotationMode),
|
||||
_mirrored, /*flip_vertical=*/false, /*flip_texture=*/true);
|
||||
_GTMDevAssert(status.ok(),
|
||||
@"render failed: %@", [NSError gus_errorWithStatus:status]);
|
||||
|
||||
glBindTexture(CVOpenGLESTextureGetTarget(texture), 0);
|
||||
glBindTexture(GL_TEXTURE_2D, 0 );
|
||||
CFRelease(texture);
|
||||
CVOpenGLESTextureCacheFlush(_textureCache, 0);
|
||||
}
|
||||
|
||||
#pragma mark - GLKViewDelegate
|
||||
|
||||
- (void)glkView:(GLKView *)view drawInRect:(CGRect)rect {
|
||||
CVPixelBufferRef pixelBuffer = NULL;
|
||||
|
||||
OSSpinLockLock(&_bufferLock);
|
||||
pixelBuffer = _nextPixelBufferToRender;
|
||||
if (_retainsLastPixelBuffer) {
|
||||
CVPixelBufferRetain(pixelBuffer);
|
||||
} else {
|
||||
_nextPixelBufferToRender = NULL;
|
||||
}
|
||||
OSSpinLockUnlock(&_bufferLock);
|
||||
|
||||
if (!pixelBuffer) return;
|
||||
|
||||
[self drawPixelBuffer:pixelBuffer width:view.drawableWidth height:view.drawableHeight];
|
||||
CVPixelBufferRelease(pixelBuffer);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <XCTest/XCTest.h>
|
||||
|
||||
#import "mediapipe/objc/MPPGraph.h"
|
||||
#import "mediapipe/objc/MPPGraphTestBase.h"
|
||||
|
||||
@interface MPPGpuSimpleTest : MPPGraphTestBase
|
||||
@end
|
||||
|
||||
@implementation MPPGpuSimpleTest{
|
||||
CFHolder<CVPixelBufferRef> _inputPixelBuffer;
|
||||
CFHolder<CVPixelBufferRef> _referencePixelBuffer;
|
||||
CFHolder<CVPixelBufferRef> _outputPixelBuffer;
|
||||
}
|
||||
- (void)setUp {
|
||||
[super setUp];
|
||||
UIImage* image = [self testImageNamed:@"sergey" extension:@"png"];
|
||||
XCTAssertTrue(CreateCVPixelBufferFromCGImage(image.CGImage, &_inputPixelBuffer).ok());
|
||||
image = [self testImageNamed:@"sobel_reference" extension:@"png"];
|
||||
XCTAssertTrue(CreateCVPixelBufferFromCGImage(image.CGImage, &_referencePixelBuffer).ok());
|
||||
}
|
||||
|
||||
// This delegate method receives output.
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
fromStream:(const std::string&)streamName
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp {
|
||||
NSLog(@"CALLBACK INVOKED");
|
||||
_outputPixelBuffer.reset(pixelBuffer);
|
||||
}
|
||||
|
||||
- (void)testSimpleGpuGraph {
|
||||
// Graph setup.
|
||||
NSData* configData = [self testDataNamed:@"test_sobel.binarypb" extension:nil];
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
XCTAssertTrue(config.ParseFromArray([configData bytes], [configData length]));
|
||||
MPPGraph* mediapipeGraph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
// We receive output by setting ourselves as the delegate.
|
||||
mediapipeGraph.delegate = self;
|
||||
[mediapipeGraph addFrameOutputStream:"output_video" outputPacketType:MediaPipePacketPixelBuffer];
|
||||
|
||||
// Start running the graph.
|
||||
NSError *error;
|
||||
BOOL success = [mediapipeGraph startWithError:&error];
|
||||
XCTAssertTrue(success, @"%@", error.localizedDescription);
|
||||
|
||||
// Send a frame.
|
||||
XCTAssertTrue([mediapipeGraph sendPixelBuffer:*_inputPixelBuffer
|
||||
intoStream:"input_video"
|
||||
packetType:MediaPipePacketPixelBuffer
|
||||
timestamp:mediapipe::Timestamp(0)]);
|
||||
|
||||
// Shut down the graph.
|
||||
success = [mediapipeGraph closeAllInputStreamsWithError:&error];
|
||||
XCTAssertTrue(success, @"%@", error.localizedDescription);
|
||||
success = [mediapipeGraph waitUntilDoneWithError:&error];
|
||||
XCTAssertTrue(success, @"%@", error.localizedDescription);
|
||||
|
||||
// Check output.
|
||||
XCTAssertTrue(_outputPixelBuffer != nullptr);
|
||||
[self savePixelBufferToSponge:*_outputPixelBuffer
|
||||
withSubpath:@"sobel.png"];
|
||||
XCTAssertTrue([self pixelBuffer:*_outputPixelBuffer
|
||||
isCloseTo:*_referencePixelBuffer
|
||||
maxLocalDifference:5
|
||||
maxAverageDifference:FLT_MAX]);
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,214 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#ifndef __cplusplus
|
||||
#error This header can only be included by an Objective-C++ file.
|
||||
#endif
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/objc/util.h"
|
||||
|
||||
@class MPPGraph;
|
||||
|
||||
namespace mediapipe {
|
||||
struct GpuSharedData;
|
||||
} // namespace mediapipe
|
||||
|
||||
/// A delegate that can receive frames from a MediaPipe graph.
|
||||
@protocol MPPGraphDelegate <NSObject>
|
||||
|
||||
/// Provides the delegate with a new video frame.
|
||||
@optional
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
fromStream:(const std::string&)streamName;
|
||||
|
||||
/// Provides the delegate with a new video frame and time stamp.
|
||||
@optional
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
fromStream:(const std::string&)streamName
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp;
|
||||
|
||||
/// Provides the delegate with a raw packet.
|
||||
@optional
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPacket:(const mediapipe::Packet&)packet
|
||||
fromStream:(const std::string&)streamName;
|
||||
|
||||
@end
|
||||
|
||||
/// Chooses the packet type used by MPPGraph to send and receive packets
|
||||
/// from the graph.
|
||||
typedef NS_ENUM(int, MediaPipePacketType) {
|
||||
/// Any packet type.
|
||||
/// Calls mediapipeGraph:didOutputPacket:fromStream:
|
||||
MediaPipePacketRaw,
|
||||
|
||||
/// CFHolder<CVPixelBufferRef>.
|
||||
/// Calls mediapipeGraph:didOutputPixelBuffer:fromStream:
|
||||
/// Use this packet type to pass GPU frames to calculators.
|
||||
MediaPipePacketPixelBuffer,
|
||||
|
||||
/// ImageFrame.
|
||||
/// Calls mediapipeGraph:didOutputPixelBuffer:fromStream:
|
||||
MediaPipePacketImageFrame,
|
||||
|
||||
/// RGBA ImageFrame, but do not swap the channels if the input pixel buffer
|
||||
/// is BGRA. This is useful when the graph needs RGBA ImageFrames, but the
|
||||
/// calculators do not care about the order of the channels, so BGRA data can
|
||||
/// be used as-is.
|
||||
/// Calls mediapipeGraph:didOutputPixelBuffer:fromStream:
|
||||
MediaPipePacketImageFrameBGRANoSwap,
|
||||
};
|
||||
|
||||
/// This class is an Objective-C wrapper around a MediaPipe graph object, and
|
||||
/// helps interface it with iOS technologies such as AVFoundation.
|
||||
@interface MPPGraph : NSObject
|
||||
|
||||
/// The delegate, which receives output frames.
|
||||
@property(weak) id<MPPGraphDelegate> delegate;
|
||||
|
||||
/// If the graph is already processing more than this number of frames, drop any
|
||||
/// new incoming frames. Used to avoid swamping slower devices when processing
|
||||
/// cannot keep up with the speed of video input.
|
||||
/// This works as long as frames are sent or received using these methods:
|
||||
/// - sendPixelBuffer:intoStream:packetType:[timestamp:]
|
||||
/// - addFrameOutputStream:outputPacketType:
|
||||
/// Set to 0 (the default) for no limit.
|
||||
@property(nonatomic) int maxFramesInFlight;
|
||||
|
||||
/// Determines whether adding a packet to an input stream whose queue is full
|
||||
/// should fail or wait.
|
||||
@property mediapipe::CalculatorGraph::GraphInputStreamAddMode packetAddMode;
|
||||
|
||||
- (instancetype)init NS_UNAVAILABLE;
|
||||
|
||||
/// Copies the config and initializes the graph.
|
||||
/// @param config The configuration describing the graph.
|
||||
- (instancetype)initWithGraphConfig:(const mediapipe::CalculatorGraphConfig&)config
|
||||
NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
- (mediapipe::ProfilingContext*)getProfiler;
|
||||
|
||||
/// Sets a stream header. If the header was already set, it is overwritten.
|
||||
/// @param packet The header.
|
||||
/// @param streamName The name of the stream.
|
||||
- (void)setHeaderPacket:(const mediapipe::Packet&)packet forStream:(const std::string&)streamName;
|
||||
|
||||
/// Sets a side packet. If it was already set, it is overwritten.
|
||||
/// Must be called before the graph is started.
|
||||
/// @param packet The packet to be associated with the input side packet.
|
||||
/// @param name The name of the input side packet.
|
||||
- (void)setSidePacket:(const mediapipe::Packet&)packet named:(const std::string&)name;
|
||||
|
||||
/// Adds input side packets from a map. Any inputs that were already set are
|
||||
/// left unchanged.
|
||||
/// Must be called before the graph is started.
|
||||
/// @param extraInputSidePackets The input side packets to be added.
|
||||
- (void)addSidePackets:(const std::map<std::string, mediapipe::Packet>&)extraSidePackets;
|
||||
|
||||
// TODO: rename to addDelegateOutputStream:packetType:
|
||||
/// Add an output stream in the graph from which the delegate wants to receive
|
||||
/// output. The delegate method called depends on the provided packetType.
|
||||
/// @param outputStreamName The name of the output stream from which
|
||||
/// the delegate will receive frames.
|
||||
/// @param packetType The type of packet provided by the output streams.
|
||||
- (void)addFrameOutputStream:(const std::string&)outputStreamName
|
||||
outputPacketType:(MediaPipePacketType)packetType;
|
||||
|
||||
/// Starts running the graph.
|
||||
/// @return YES if successful.
|
||||
- (BOOL)startWithError:(NSError**)error;
|
||||
|
||||
/// Sends a generic packet into a graph input stream.
|
||||
/// The graph must have been started before calling this.
|
||||
/// Returns YES if the packet was successfully sent.
|
||||
- (BOOL)sendPacket:(const mediapipe::Packet&)packet
|
||||
intoStream:(const std::string&)streamName
|
||||
error:(NSError**)error;
|
||||
|
||||
- (BOOL)movePacket:(mediapipe::Packet&&)packet
|
||||
intoStream:(const std::string&)streamName
|
||||
error:(NSError**)error;
|
||||
|
||||
/// Sets the maximum queue size for a stream. Experimental feature, currently
|
||||
/// only supported for graph input streams. Should be called before starting the
|
||||
/// graph.
|
||||
- (BOOL)setMaxQueueSize:(int)maxQueueSize
|
||||
forStream:(const std::string&)streamName
|
||||
error:(NSError**)error;
|
||||
|
||||
/// Creates a MediaPipe packet wrapping the given pixelBuffer;
|
||||
- (mediapipe::Packet)packetWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
packetType:(MediaPipePacketType)packetType;
|
||||
|
||||
/// Sends a pixel buffer into a graph input stream, using the specified packet
|
||||
/// type. The graph must have been started before calling this. Drops frames and
|
||||
/// returns NO if maxFramesInFlight is exceeded. If allowOverwrite is set to YES,
|
||||
/// allows MediaPipe to overwrite the packet contents on successful sending for
|
||||
/// possibly increased efficiency. Returns YES if the packet was successfully sent.
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp
|
||||
allowOverwrite:(BOOL)allowOverwrite;
|
||||
|
||||
/// Sends a pixel buffer into a graph input stream, using the specified packet
|
||||
/// type. The graph must have been started before calling this. Drops frames and
|
||||
/// returns NO if maxFramesInFlight is exceeded. Returns YES if the packet was
|
||||
/// successfully sent.
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp;
|
||||
|
||||
/// Sends a pixel buffer into a graph input stream, using the specified packet
|
||||
/// type. The graph must have been started before calling this. The timestamp is
|
||||
/// automatically incremented from the last timestamp used by this method. Drops
|
||||
/// frames and returns NO if maxFramesInFlight is exceeded. Returns YES if the
|
||||
/// packet was successfully sent.
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType;
|
||||
|
||||
/// Cancels a graph run. You must still call waitUntilDoneWithError: after this.
|
||||
- (void)cancel;
|
||||
|
||||
/// Check if the graph contains this input stream
|
||||
- (BOOL)hasInputStream:(const std::string&)inputName;
|
||||
|
||||
/// Closes an input stream.
|
||||
/// You must close all graph input streams before stopping the graph.
|
||||
/// @return YES if successful.
|
||||
- (BOOL)closeInputStream:(const std::string&)inputName error:(NSError**)error;
|
||||
|
||||
/// Closes all graph input streams.
|
||||
/// @return YES if successful.
|
||||
- (BOOL)closeAllInputStreamsWithError:(NSError**)error;
|
||||
|
||||
/// Stops running the graph.
|
||||
/// Call this before releasing this object. All input streams must have been
|
||||
/// closed. This call does not time out, so you should not call it from the main
|
||||
/// thread.
|
||||
/// @return YES if successful.
|
||||
- (BOOL)waitUntilDoneWithError:(NSError**)error;
|
||||
|
||||
/// Waits for the graph to become idle.
|
||||
- (BOOL)waitUntilIdleWithError:(NSError**)error;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,373 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/objc/MPPGraph.h"
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Accelerate/Accelerate.h>
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/gpu/MPPGraphGPUData.h"
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
#include "mediapipe/gpu/gpu_shared_data_internal.h"
|
||||
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
#import "GTMDefines.h"
|
||||
|
||||
@implementation MPPGraph {
|
||||
// Graph is wrapped in a unique_ptr because it was generating 39+KB of unnecessary ObjC runtime
|
||||
// information. See https://medium.com/@dmaclach/objective-c-encoding-and-you-866624cc02de
|
||||
// for details.
|
||||
std::unique_ptr<mediapipe::CalculatorGraph> _graph;
|
||||
/// Input side packets that will be added to the graph when it is started.
|
||||
std::map<std::string, mediapipe::Packet> _inputSidePackets;
|
||||
/// Packet headers that will be added to the graph when it is started.
|
||||
std::map<std::string, mediapipe::Packet> _streamHeaders;
|
||||
|
||||
/// Number of frames currently being processed by the graph.
|
||||
std::atomic<int32_t> _framesInFlight;
|
||||
/// Used as a sequential timestamp for MediaPipe.
|
||||
mediapipe::Timestamp _frameTimestamp;
|
||||
int64 _frameNumber;
|
||||
|
||||
// Graph config modified to expose requested output streams.
|
||||
mediapipe::CalculatorGraphConfig _config;
|
||||
|
||||
// Tracks whether the graph has been started and is currently running.
|
||||
BOOL _started;
|
||||
}
|
||||
|
||||
- (instancetype)initWithGraphConfig:(const mediapipe::CalculatorGraphConfig&)config {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
// Turn on Cocoa multithreading, since MediaPipe uses threads.
|
||||
// Not needed on iOS, but we may want to have OS X clients in the future.
|
||||
[[[NSThread alloc] init] start];
|
||||
_graph = absl::make_unique<mediapipe::CalculatorGraph>();
|
||||
_config = config;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (mediapipe::ProfilingContext*)getProfiler {
|
||||
return _graph->profiler();
|
||||
}
|
||||
|
||||
- (mediapipe::CalculatorGraph::GraphInputStreamAddMode)packetAddMode {
|
||||
return _graph->GetGraphInputStreamAddMode();
|
||||
}
|
||||
|
||||
- (void)setPacketAddMode:(mediapipe::CalculatorGraph::GraphInputStreamAddMode)mode {
|
||||
_graph->SetGraphInputStreamAddMode(mode);
|
||||
}
|
||||
|
||||
- (void)addFrameOutputStream:(const std::string&)outputStreamName
|
||||
outputPacketType:(MediaPipePacketType)packetType {
|
||||
std::string callbackInputName;
|
||||
mediapipe::tool::AddCallbackCalculator(outputStreamName, &_config, &callbackInputName,
|
||||
/*use_std_function=*/true);
|
||||
// No matter what ownership qualifiers are put on the pointer, NewPermanentCallback will
|
||||
// still end up with a strong pointer to MPPGraph*. That is why we use void* instead.
|
||||
void* wrapperVoid = (__bridge void*)self;
|
||||
_inputSidePackets[callbackInputName] =
|
||||
mediapipe::MakePacket<std::function<void(const mediapipe::Packet&)>>(
|
||||
[wrapperVoid, outputStreamName, packetType](const mediapipe::Packet& packet) {
|
||||
CallFrameDelegate(wrapperVoid, outputStreamName, packetType, packet);
|
||||
});
|
||||
}
|
||||
|
||||
- (NSString *)description {
|
||||
return [NSString stringWithFormat:@"<%@: %p; framesInFlight = %d>", [self class], self,
|
||||
_framesInFlight.load(std::memory_order_relaxed)];
|
||||
}
|
||||
|
||||
/// This is the function that gets called by the CallbackCalculator that
|
||||
/// receives the graph's output.
|
||||
void CallFrameDelegate(void* wrapperVoid, const std::string& streamName,
|
||||
MediaPipePacketType packetType, const mediapipe::Packet& packet) {
|
||||
MPPGraph* wrapper = (__bridge MPPGraph*)wrapperVoid;
|
||||
@autoreleasepool {
|
||||
if (packetType == MediaPipePacketRaw) {
|
||||
[wrapper.delegate mediapipeGraph:wrapper
|
||||
didOutputPacket:packet
|
||||
fromStream:streamName];
|
||||
} else if (packetType == MediaPipePacketImageFrame) {
|
||||
const auto& frame = packet.Get<mediapipe::ImageFrame>();
|
||||
mediapipe::ImageFormat::Format format = frame.Format();
|
||||
|
||||
if (format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::GRAY8) {
|
||||
CVPixelBufferRef pixelBuffer;
|
||||
// To ensure compatibility with CVOpenGLESTextureCache, this attribute should be present.
|
||||
NSDictionary* attributes = @{
|
||||
(id)kCVPixelBufferIOSurfacePropertiesKey : @{},
|
||||
};
|
||||
// If kCVPixelFormatType_32RGBA does not work, it returns kCVReturnInvalidPixelFormat.
|
||||
CVReturn error = CVPixelBufferCreate(
|
||||
NULL, frame.Width(), frame.Height(), kCVPixelFormatType_32BGRA,
|
||||
(__bridge CFDictionaryRef)attributes, &pixelBuffer);
|
||||
_GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferCreate failed: %d", error);
|
||||
error = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
_GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferLockBaseAddress failed: %d", error);
|
||||
|
||||
vImage_Buffer vDestination = vImageForCVPixelBuffer(pixelBuffer);
|
||||
// Note: we have to throw away const here, but we should not overwrite
|
||||
// the packet data.
|
||||
vImage_Buffer vSource = vImageForImageFrame(frame);
|
||||
if (format == mediapipe::ImageFormat::SRGBA) {
|
||||
// Swap R and B channels.
|
||||
const uint8_t permuteMap[4] = {2, 1, 0, 3};
|
||||
vImage_Error vError = vImagePermuteChannels_ARGB8888(
|
||||
&vSource, &vDestination, permuteMap, kvImageNoFlags);
|
||||
_GTMDevAssert(vError == kvImageNoError, @"vImagePermuteChannels failed: %zd", vError);
|
||||
} else {
|
||||
// Convert grayscale back to BGRA
|
||||
vImage_Error vError = vImageGrayToBGRA(&vSource, &vDestination);
|
||||
_GTMDevAssert(vError == kvImageNoError, @"vImageGrayToBGRA failed: %zd", vError);
|
||||
}
|
||||
|
||||
error = CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
_GTMDevAssert(error == kCVReturnSuccess,
|
||||
@"CVPixelBufferUnlockBaseAddress failed: %d", error);
|
||||
|
||||
if ([wrapper.delegate respondsToSelector:@selector
|
||||
(mediapipeGraph:didOutputPixelBuffer:fromStream:timestamp:)]) {
|
||||
[wrapper.delegate mediapipeGraph:wrapper
|
||||
didOutputPixelBuffer:pixelBuffer
|
||||
fromStream:streamName
|
||||
timestamp:packet.Timestamp()];
|
||||
} else if ([wrapper.delegate respondsToSelector:@selector
|
||||
(mediapipeGraph:didOutputPixelBuffer:fromStream:)]) {
|
||||
[wrapper.delegate mediapipeGraph:wrapper
|
||||
didOutputPixelBuffer:pixelBuffer
|
||||
fromStream:streamName];
|
||||
}
|
||||
CVPixelBufferRelease(pixelBuffer);
|
||||
} else {
|
||||
_GTMDevLog(@"unsupported ImageFormat: %d", format);
|
||||
}
|
||||
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
} else if (packetType == MediaPipePacketPixelBuffer) {
|
||||
CVPixelBufferRef pixelBuffer = packet.Get<mediapipe::GpuBuffer>().GetCVPixelBufferRef();
|
||||
if ([wrapper.delegate
|
||||
respondsToSelector:@selector
|
||||
(mediapipeGraph:didOutputPixelBuffer:fromStream:timestamp:)]) {
|
||||
[wrapper.delegate mediapipeGraph:wrapper
|
||||
didOutputPixelBuffer:pixelBuffer
|
||||
fromStream:streamName
|
||||
timestamp:packet.Timestamp()];
|
||||
} else if ([wrapper.delegate
|
||||
respondsToSelector:@selector
|
||||
(mediapipeGraph:didOutputPixelBuffer:fromStream:)]) {
|
||||
[wrapper.delegate mediapipeGraph:wrapper
|
||||
didOutputPixelBuffer:pixelBuffer
|
||||
fromStream:streamName];
|
||||
}
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
} else {
|
||||
_GTMDevLog(@"unsupported packet type");
|
||||
}
|
||||
|
||||
wrapper->_framesInFlight--;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setHeaderPacket:(const mediapipe::Packet&)packet forStream:(const std::string&)streamName {
|
||||
_GTMDevAssert(!_started, @"%@ must be called before the graph is started",
|
||||
NSStringFromSelector(_cmd));
|
||||
_streamHeaders[streamName] = packet;
|
||||
}
|
||||
|
||||
- (void)setSidePacket:(const mediapipe::Packet&)packet named:(const std::string&)name {
|
||||
_GTMDevAssert(!_started, @"%@ must be called before the graph is started",
|
||||
NSStringFromSelector(_cmd));
|
||||
_inputSidePackets[name] = packet;
|
||||
}
|
||||
|
||||
- (void)addSidePackets:(const std::map<std::string, mediapipe::Packet>&)extraSidePackets {
|
||||
_GTMDevAssert(!_started, @"%@ must be called before the graph is started",
|
||||
NSStringFromSelector(_cmd));
|
||||
_inputSidePackets.insert(extraSidePackets.begin(), extraSidePackets.end());
|
||||
}
|
||||
|
||||
- (BOOL)startWithError:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->Initialize(_config);
|
||||
if (status.ok()) {
|
||||
status = _graph->StartRun(_inputSidePackets, _streamHeaders);
|
||||
if (status.ok()) {
|
||||
_started = YES;
|
||||
return YES;
|
||||
}
|
||||
}
|
||||
if (error) {
|
||||
*error = [NSError gus_errorWithStatus:status];
|
||||
}
|
||||
return NO;
|
||||
}
|
||||
|
||||
- (void)cancel {
|
||||
_graph->Cancel();
|
||||
}
|
||||
|
||||
- (BOOL)hasInputStream:(const std::string&)inputName {
|
||||
return _graph->HasInputStream(inputName);
|
||||
}
|
||||
|
||||
- (BOOL)closeInputStream:(const std::string&)inputName error:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->CloseInputStream(inputName);
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)closeAllInputStreamsWithError:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->CloseAllInputStreams();
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)waitUntilDoneWithError:(NSError**)error {
|
||||
// Since this method blocks with no timeout, it should not be called in the main thread in
|
||||
// an app. However, it's fine to allow that in a test.
|
||||
// TODO: is this too heavy-handed? Maybe a warning would be fine.
|
||||
_GTMDevAssert(![NSThread isMainThread] || (NSClassFromString(@"XCTest")),
|
||||
@"waitUntilDoneWithError: should not be called on the main thread");
|
||||
::mediapipe::Status status = _graph->WaitUntilDone();
|
||||
_started = NO;
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)waitUntilIdleWithError:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->WaitUntilIdle();
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)movePacket:(mediapipe::Packet&&)packet
|
||||
intoStream:(const std::string&)streamName
|
||||
error:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->AddPacketToInputStream(streamName, std::move(packet));
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)sendPacket:(const mediapipe::Packet&)packet
|
||||
intoStream:(const std::string&)streamName
|
||||
error:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->AddPacketToInputStream(streamName, packet);
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (BOOL)setMaxQueueSize:(int)maxQueueSize
|
||||
forStream:(const std::string&)streamName
|
||||
error:(NSError**)error {
|
||||
::mediapipe::Status status = _graph->SetInputStreamMaxQueueSize(streamName, maxQueueSize);
|
||||
if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status];
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
- (mediapipe::Packet)packetWithPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
packetType:(MediaPipePacketType)packetType {
|
||||
mediapipe::Packet packet;
|
||||
if (packetType == MediaPipePacketImageFrame || packetType == MediaPipePacketImageFrameBGRANoSwap) {
|
||||
auto frame = CreateImageFrameForCVPixelBuffer(
|
||||
imageBuffer, /* canOverwrite = */ false,
|
||||
/* bgrAsRgb = */ packetType == MediaPipePacketImageFrameBGRANoSwap);
|
||||
packet = mediapipe::Adopt(frame.release());
|
||||
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
} else if (packetType == MediaPipePacketPixelBuffer) {
|
||||
packet = mediapipe::MakePacket<mediapipe::GpuBuffer>(imageBuffer);
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
} else {
|
||||
_GTMDevLog(@"unsupported packet type: %d", packetType);
|
||||
}
|
||||
return packet;
|
||||
}
|
||||
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp
|
||||
allowOverwrite:(BOOL)allowOverwrite {
|
||||
if (_maxFramesInFlight && _framesInFlight >= _maxFramesInFlight) return NO;
|
||||
mediapipe::Packet packet = [self packetWithPixelBuffer:imageBuffer packetType:packetType];
|
||||
NSError* error;
|
||||
BOOL success;
|
||||
if (allowOverwrite) {
|
||||
packet = std::move(packet).At(timestamp);
|
||||
success = [self movePacket:std::move(packet)
|
||||
intoStream:inputName
|
||||
error:&error];
|
||||
} else {
|
||||
success = [self sendPacket:packet.At(timestamp)
|
||||
intoStream:inputName
|
||||
error:&error];
|
||||
}
|
||||
if (success) _framesInFlight++;
|
||||
else _GTMDevLog(@"failed to send packet: %@", error);
|
||||
return success;
|
||||
}
|
||||
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType
|
||||
timestamp:(const mediapipe::Timestamp&)timestamp {
|
||||
return [self sendPixelBuffer:imageBuffer
|
||||
intoStream:inputName
|
||||
packetType:packetType
|
||||
timestamp:timestamp
|
||||
allowOverwrite:NO];
|
||||
}
|
||||
|
||||
- (BOOL)sendPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
intoStream:(const std::string&)inputName
|
||||
packetType:(MediaPipePacketType)packetType {
|
||||
_GTMDevAssert(_frameTimestamp < mediapipe::Timestamp::Done(),
|
||||
@"Trying to send frame after stream is done.");
|
||||
if (_frameTimestamp < mediapipe::Timestamp::Min()) {
|
||||
_frameTimestamp = mediapipe::Timestamp::Min();
|
||||
} else {
|
||||
_frameTimestamp++;
|
||||
}
|
||||
return [self sendPixelBuffer:imageBuffer
|
||||
intoStream:inputName
|
||||
packetType:packetType
|
||||
timestamp:_frameTimestamp];
|
||||
}
|
||||
|
||||
- (void)debugPrintGlInfo {
|
||||
std::shared_ptr<mediapipe::GpuResources> gpu_resources = _graph->GetGpuResources();
|
||||
if (!gpu_resources) {
|
||||
NSLog(@"GPU not set up.");
|
||||
return;
|
||||
}
|
||||
|
||||
NSString* extensionString;
|
||||
(void)gpu_resources->gl_context()->Run([&extensionString]{
|
||||
extensionString = [NSString stringWithUTF8String:(char*)glGetString(GL_EXTENSIONS)];
|
||||
return ::mediapipe::OkStatus();
|
||||
});
|
||||
|
||||
NSArray* extensions = [extensionString componentsSeparatedByCharactersInSet:
|
||||
[NSCharacterSet whitespaceCharacterSet]];
|
||||
for (NSString* oneExtension in extensions)
|
||||
NSLog(@"%@", oneExtension);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <XCTest/XCTest.h>
|
||||
|
||||
#import "mediapipe/objc/MPPGraph.h"
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
#import "mediapipe/objc/util.h"
|
||||
|
||||
/// This XCTestCase subclass provides common convenience methods for testing
|
||||
/// with MPPGraph.
|
||||
@interface MPPGraphTestBase : XCTestCase <MPPGraphDelegate> {
|
||||
/// This block is used to respond to mediapipeGraph:didOutputPixelBuffer:fromStream:.
|
||||
/// runGraph:withPixelBuffer:packetType: uses this internally, but you can reuse it
|
||||
/// if you need to run a graph directly and want a MPPGraphTestBase object to
|
||||
/// act as the delegate.
|
||||
void (^_pixelBufferOutputBlock)(MPPGraph* graph, CVPixelBufferRef imageBuffer,
|
||||
const std::string& streamName);
|
||||
|
||||
/// This block is used to respond to mediapipeGraph:didOutputPacket:fromStream:.
|
||||
/// You can use it if you need to run a graph directly and want a MPPGraphTestBase
|
||||
/// object to act as the delegate.
|
||||
void (^_packetOutputBlock)(MPPGraph* graph, const mediapipe::Packet& packet,
|
||||
const std::string& streamName);
|
||||
}
|
||||
|
||||
/// Runs a single frame through a simple graph. The graph is expected to have an
|
||||
/// input stream named "input_frames" and an output stream named
|
||||
/// "output_frames". This function runs the graph, sends inputBuffer into
|
||||
/// input_frames (at timestamp=1), receives an output buffer from output_frames,
|
||||
/// completes the run, and returns the output frame.
|
||||
- (CVPixelBufferRef)runGraph:(MPPGraph*)graph
|
||||
withPixelBuffer:(CVPixelBufferRef)inputBuffer
|
||||
packetType:(MediaPipePacketType)inputPacketType;
|
||||
|
||||
/// Runs a simple graph, providing a single frame to zero or more inputs. Input images are wrapped
|
||||
/// in packets each with timestamp mediapipe::Timestamp(1). Those packets are added to the
|
||||
/// designated streams (named by the keys of withInputPixelBuffers). When a packet arrives on the
|
||||
/// output stream, the graph run is done and the output frame is returned.
|
||||
- (CVPixelBufferRef)runGraph:(MPPGraph*)graph
|
||||
withInputPixelBuffers:
|
||||
(const std::unordered_map<std::string, CFHolder<CVPixelBufferRef>>&)inputBuffers
|
||||
outputStream:(const std::string&)output
|
||||
packetType:(MediaPipePacketType)inputPacketType;
|
||||
|
||||
/// Loads a data file from the test bundle.
|
||||
- (NSData*)testDataNamed:(NSString*)name extension:(NSString*)extension;
|
||||
|
||||
/// Loads an image from the test bundle.
|
||||
- (UIImage*)testImageNamed:(NSString*)name extension:(NSString*)extension;
|
||||
|
||||
/// Loads an image from the test bundle in subpath.
|
||||
- (UIImage*)testImageNamed:(NSString*)name
|
||||
extension:(NSString*)extension
|
||||
subdirectory:(NSString*)subdirectory;
|
||||
|
||||
/// Compares two pixel buffers for strict equality.
|
||||
/// Returns true iff the two buffers have the same size, format, and pixel data.
|
||||
- (BOOL)pixelBuffer:(CVPixelBufferRef)a isEqualTo:(CVPixelBufferRef)b;
|
||||
|
||||
/// Compares two pixel buffers with some leniency.
|
||||
/// Returns true iff the two buffers have the same size and format, and:
|
||||
/// - the difference between each pixel of A and the corresponding pixel of B does
|
||||
/// not exceed maxLocalDiff, and
|
||||
/// - the average difference between corresponding pixels of A and B does not
|
||||
/// exceed maxAvgDiff.
|
||||
- (BOOL)pixelBuffer:(CVPixelBufferRef)a
|
||||
isCloseTo:(CVPixelBufferRef)b
|
||||
maxLocalDifference:(int)maxLocalDiff
|
||||
maxAverageDifference:(float)maxAvgDiff;
|
||||
|
||||
/// Utility function for making a copy of a pixel buffer with a different pixel
|
||||
/// format.
|
||||
- (CVPixelBufferRef)convertPixelBuffer:(CVPixelBufferRef)input toPixelFormat:(OSType)pixelFormat;
|
||||
|
||||
/// Makes a scaled copy of a BGRA pixel buffer.
|
||||
- (CVPixelBufferRef)scaleBGRAPixelBuffer:(CVPixelBufferRef)input toSize:(CGSize)size;
|
||||
|
||||
/// Utility function for transforming a pixel buffer.
|
||||
/// It creates a new pixel buffer with the same dimensions as the original, in the
|
||||
/// desired pixel format, and invokes a block with the input and output buffers.
|
||||
/// The buffers are locked before the block and unlocked after, so the block can read
|
||||
/// from the input buffer and write to the output buffer without further preparation.
|
||||
- (CVPixelBufferRef)transformPixelBuffer:(CVPixelBufferRef)input
|
||||
outputPixelFormat:(OSType)pixelFormat
|
||||
transformation:(void (^)(CVPixelBufferRef input,
|
||||
CVPixelBufferRef output))transformation;
|
||||
|
||||
/// Computes a difference image from two input images. Useful for debugging.
|
||||
- (UIImage*)differenceOfImage:(UIImage*)inputA image:(UIImage*)inputB;
|
||||
|
||||
/// Tests a graph by sending in the provided input pixel buffer and comparing the
|
||||
/// output with the provided expected output. Uses runGraph:withPixelBuffer:packetType:
|
||||
/// internally, so the streams are supposed to be named "input_frames" and "output_frames".
|
||||
/// The actual and expected outputs are compared fuzzily.
|
||||
- (void)testGraph:(MPPGraph*)graph
|
||||
input:(CVPixelBufferRef)inputBuffer
|
||||
expectedOutput:(CVPixelBufferRef)expectedBuffer;
|
||||
|
||||
/// Tests a graph by sending the provided image files as pixelBuffer inputs to the
|
||||
/// corresponding streams, and comparing the single frame output by the given output stream
|
||||
/// with the contents of the given output file.
|
||||
/// @param config Graph config.
|
||||
/// @param fileInputs Dictionary mapping input stream names to image file paths.
|
||||
/// @param packetInputs Map of input stream names to additional input packets.
|
||||
/// @param sidePackets Map of input side packet stream names to packets.
|
||||
/// @param outputStream Name of the output stream where the output is produced.
|
||||
/// @param expectedPath Path to an image file containing the expected output.
|
||||
/// @param maxAverageDifference The maximum allowable average pixel difference
|
||||
/// between the
|
||||
/// expected output and computed output.
|
||||
/// TODO: Use NSDictionary instead of std::map for sidePackets.
|
||||
- (void)testGraphConfig:(const mediapipe::CalculatorGraphConfig&)config
|
||||
inputStreamsAndFiles:(NSDictionary<NSString*, NSString*>*)fileInputs
|
||||
inputStreamsAndPackets:(const std::map<std::string, mediapipe::Packet>&)packetInputs
|
||||
sidePackets:(std::map<std::string, mediapipe::Packet>)sidePackets
|
||||
timestamp:(mediapipe::Timestamp)timestamp
|
||||
outputStream:(NSString*)outputStream
|
||||
expectedOutputFile:(NSString*)expectedPath
|
||||
maxAverageDifference:(float)maxAverageDifference;
|
||||
|
||||
/// Calls the above testGraphConfig: method with a default maxAverageDifference
|
||||
/// of 1.f and timestamp of 1.
|
||||
- (void)testGraphConfig:(const mediapipe::CalculatorGraphConfig&)config
|
||||
inputStreamsAndFiles:(NSDictionary<NSString*, NSString*>*)inputs
|
||||
outputStream:(NSString*)outputStream
|
||||
expectedOutputFile:(NSString*)expectedPath;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,446 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/objc/MPPGraphTestBase.h"
|
||||
#import "mediapipe/objc/Weakify.h"
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
|
||||
static UIImage* UIImageWithPixelBuffer(CVPixelBufferRef pixelBuffer) {
|
||||
CFHolder<CGImageRef> cgImage;
|
||||
::mediapipe::Status status = CreateCGImageFromCVPixelBuffer(pixelBuffer, &cgImage);
|
||||
if (!status.ok()) {
|
||||
return nil;
|
||||
}
|
||||
UIImage *uiImage = [UIImage imageWithCGImage:*cgImage
|
||||
scale:1.0
|
||||
orientation:UIImageOrientationUp];
|
||||
return uiImage;
|
||||
}
|
||||
|
||||
static void EnsureOutputDirFor(NSString *outputFile) {
|
||||
NSFileManager *fileManager = [NSFileManager defaultManager];
|
||||
NSError *error = nil;
|
||||
BOOL result = [fileManager createDirectoryAtPath:[outputFile stringByDeletingLastPathComponent]
|
||||
withIntermediateDirectories:YES
|
||||
attributes:nil
|
||||
error:&error];
|
||||
// TODO: Log the error for clarity. The file-write will fail later
|
||||
// but it would be nice to see this error. However, 'error' is still testing
|
||||
// false and result is true even on an unwritable path-- not sure what's up.
|
||||
}
|
||||
|
||||
@implementation MPPGraphTestBase
|
||||
|
||||
- (NSData*)testDataNamed:(NSString*)name extension:(NSString*)extension {
|
||||
NSBundle* testBundle = [NSBundle bundleForClass:[self class]];
|
||||
NSURL* resourceURL = [testBundle URLForResource:name withExtension:extension];
|
||||
XCTAssertNotNil(resourceURL,
|
||||
@"Unable to find data with name: %@. Did you add it to your resources?", name);
|
||||
NSError* error;
|
||||
NSData* data = [NSData dataWithContentsOfURL:resourceURL options:0 error:&error];
|
||||
XCTAssertNotNil(data, @"%@: %@", resourceURL.path, error);
|
||||
return data;
|
||||
}
|
||||
|
||||
- (UIImage*)testImageNamed:(NSString*)name extension:(NSString*)extension {
|
||||
return [self testImageNamed:name extension:extension subdirectory:nil];
|
||||
}
|
||||
|
||||
- (UIImage*)testImageNamed:(NSString*)name
|
||||
extension:(NSString*)extension
|
||||
subdirectory:(NSString *)subdirectory {
|
||||
// imageNamed does not work in our test bundle
|
||||
NSBundle* testBundle = [NSBundle bundleForClass:[self class]];
|
||||
NSURL* imageURL = subdirectory ?
|
||||
[testBundle URLForResource:name withExtension:extension subdirectory:subdirectory] :
|
||||
[testBundle URLForResource:name withExtension:extension];
|
||||
XCTAssertNotNil(imageURL,
|
||||
@"Unable to find image with name: %@. Did you add it to your resources?", name);
|
||||
NSError* error;
|
||||
NSData* imageData = [NSData dataWithContentsOfURL:imageURL options:0 error:&error];
|
||||
UIImage* image = [UIImage imageWithData:imageData];
|
||||
XCTAssertNotNil(image, @"%@: %@", imageURL.path, error);
|
||||
return image;
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)runGraph:(MPPGraph*)graph
|
||||
withInputPixelBuffers:
|
||||
(const std::unordered_map<std::string, CFHolder<CVPixelBufferRef>>&)inputBuffers
|
||||
inputPackets:(const std::map<std::string, mediapipe::Packet>&)inputPackets
|
||||
timestamp:(mediapipe::Timestamp)timestamp
|
||||
outputStream:(const std::string&)outputStream
|
||||
packetType:(MediaPipePacketType)inputPacketType {
|
||||
__block CVPixelBufferRef output;
|
||||
graph.delegate = self;
|
||||
|
||||
// The XCTAssert macros contain references to self, which causes a retain cycle,
|
||||
// since the block retains self and self retains the block. The cycle is broken
|
||||
// at the end of this method, with _pixelBufferOutputBlock = nil, but Clang does
|
||||
// not realize that and outputs a warning. WEAKIFY and STRONGIFY, though not
|
||||
// strictly necessary, are used here to avoid the warning.
|
||||
WEAKIFY(self);
|
||||
if (!_pixelBufferOutputBlock) {
|
||||
XCTestExpectation* outputReceived = [self expectationWithDescription:@"output received"];
|
||||
_pixelBufferOutputBlock = ^(MPPGraph* outputGraph, CVPixelBufferRef outputBuffer,
|
||||
const std::string& outputStreamName) {
|
||||
STRONGIFY(self);
|
||||
XCTAssertEqualObjects(outputGraph, graph);
|
||||
XCTAssertEqual(outputStreamName, outputStream);
|
||||
CFRetain(outputBuffer);
|
||||
output = outputBuffer;
|
||||
[outputReceived fulfill];
|
||||
};
|
||||
}
|
||||
|
||||
NSError *error;
|
||||
BOOL success = [graph startWithError:&error];
|
||||
// Normally we continue after failures, but there is no sense in waiting for an
|
||||
// output if the graph didn't even start.
|
||||
BOOL savedContinue = self.continueAfterFailure;
|
||||
self.continueAfterFailure = NO;
|
||||
XCTAssert(success, @"%@", error.localizedDescription);
|
||||
self.continueAfterFailure = savedContinue;
|
||||
for (const auto& stream_buffer : inputBuffers) {
|
||||
[graph sendPixelBuffer:*stream_buffer.second
|
||||
intoStream:stream_buffer.first
|
||||
packetType:inputPacketType
|
||||
timestamp:timestamp];
|
||||
success = [graph closeInputStream:stream_buffer.first error:&error];
|
||||
XCTAssert(success, @"%@", error.localizedDescription);
|
||||
}
|
||||
for (const auto& stream_packet : inputPackets) {
|
||||
[graph sendPacket:stream_packet.second
|
||||
intoStream:stream_packet.first
|
||||
error:&error];
|
||||
success = [graph closeInputStream:stream_packet.first error:&error];
|
||||
XCTAssert(success, @"%@", error.localizedDescription);
|
||||
}
|
||||
|
||||
XCTestExpectation* graphDone = [self expectationWithDescription:@"graph done"];
|
||||
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
|
||||
NSError *error;
|
||||
BOOL success = [graph waitUntilDoneWithError:&error];
|
||||
XCTAssert(success, @"%@", error.localizedDescription);
|
||||
[graphDone fulfill];
|
||||
});
|
||||
|
||||
[self waitForExpectationsWithTimeout:8.0 handler:NULL];
|
||||
_pixelBufferOutputBlock = nil;
|
||||
return output;
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)runGraph:(MPPGraph*)graph
|
||||
withPixelBuffer:(CVPixelBufferRef)inputBuffer
|
||||
packetType:(MediaPipePacketType)inputPacketType {
|
||||
return [self runGraph:graph
|
||||
withInputPixelBuffers:{{"input_frames", MakeCFHolder(inputBuffer)}}
|
||||
inputPackets:{}
|
||||
timestamp:mediapipe::Timestamp(1)
|
||||
outputStream:"output_frames"
|
||||
packetType:inputPacketType];
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)runGraph:(MPPGraph*)graph
|
||||
withInputPixelBuffers:
|
||||
(const std::unordered_map<std::string, CFHolder<CVPixelBufferRef>>&)inputBuffers
|
||||
outputStream:(const std::string&)output
|
||||
packetType:(MediaPipePacketType)inputPacketType {
|
||||
return [self runGraph:graph
|
||||
withInputPixelBuffers:inputBuffers
|
||||
inputPackets:{}
|
||||
timestamp:mediapipe::Timestamp(1)
|
||||
outputStream:output
|
||||
packetType:inputPacketType];
|
||||
}
|
||||
|
||||
// By using a block to handle the delegate message, we can change the
|
||||
// implementation for each test.
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPixelBuffer:(CVPixelBufferRef)imageBuffer
|
||||
fromStream:(const std::string&)streamName {
|
||||
_pixelBufferOutputBlock(graph, imageBuffer, streamName);
|
||||
}
|
||||
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPacket:(const mediapipe::Packet&)packet
|
||||
fromStream:(const std::string&)streamName {
|
||||
_packetOutputBlock(graph, packet, streamName);
|
||||
}
|
||||
|
||||
- (BOOL)pixelBuffer:(CVPixelBufferRef)a isEqualTo:(CVPixelBufferRef)b {
|
||||
return [self pixelBuffer:a isCloseTo:b maxLocalDifference:0 maxAverageDifference:0];
|
||||
}
|
||||
|
||||
- (BOOL)pixelBuffer:(CVPixelBufferRef)a isCloseTo:(CVPixelBufferRef)b
|
||||
maxLocalDifference:(int)maxLocalDiff maxAverageDifference:(float)maxAvgDiff {
|
||||
size_t aBytesPerRow = CVPixelBufferGetBytesPerRow(a);
|
||||
size_t aWidth = CVPixelBufferGetWidth(a);
|
||||
size_t aHeight = CVPixelBufferGetHeight(a);
|
||||
OSType aPixelFormat = CVPixelBufferGetPixelFormatType(a);
|
||||
XCTAssertFalse(CVPixelBufferIsPlanar(a), @"planar buffers not supported");
|
||||
|
||||
size_t bBytesPerRow = CVPixelBufferGetBytesPerRow(b);
|
||||
size_t bWidth = CVPixelBufferGetWidth(b);
|
||||
size_t bHeight = CVPixelBufferGetHeight(b);
|
||||
OSType bPixelFormat = CVPixelBufferGetPixelFormatType(b);
|
||||
XCTAssertFalse(CVPixelBufferIsPlanar(b), @"planar buffers not supported");
|
||||
|
||||
if (aPixelFormat != bPixelFormat ||
|
||||
aWidth != bWidth ||
|
||||
aHeight != bHeight) return NO;
|
||||
|
||||
size_t bytesPerPixel; // is there a generic way to get this from a pixel buffer?
|
||||
switch (aPixelFormat) {
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
bytesPerPixel = 4;
|
||||
break;
|
||||
case kCVPixelFormatType_OneComponent8:
|
||||
bytesPerPixel = 1;
|
||||
break;
|
||||
default:
|
||||
XCTFail(@"unsupported pixel format");
|
||||
}
|
||||
|
||||
CVReturn err;
|
||||
err = CVPixelBufferLockBaseAddress(a, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(err, kCVReturnSuccess);
|
||||
err = CVPixelBufferLockBaseAddress(b, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(err, kCVReturnSuccess);
|
||||
const uint8_t* aData = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(a));
|
||||
const uint8_t* bData = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(b));
|
||||
|
||||
// Let's not assume identical bytesPerRow. Also, the padding may not be equal
|
||||
// even if bytesPerRow match.
|
||||
size_t usedRowWidth = aWidth * bytesPerPixel;
|
||||
BOOL equal = YES;
|
||||
float averageDiff = 0;
|
||||
float count = 0;
|
||||
for (int i = aHeight; i > 0 && equal; --i) {
|
||||
if (maxLocalDiff == 0) {
|
||||
// If we can, use memcmp for speed.
|
||||
equal = memcmp(aData, bData, usedRowWidth) == 0;
|
||||
} else {
|
||||
for (int j = 0; j < usedRowWidth; j++) {
|
||||
int diff = abs(aData[j] - bData[j]);
|
||||
if (diff > maxLocalDiff) {
|
||||
equal = NO;
|
||||
break;
|
||||
}
|
||||
// We use Welford's algorithm for computing a sample mean. This has better
|
||||
// numerical stability than the naive method, as noted in TAoCP. Not that it
|
||||
// particularly matters here.
|
||||
// Welford: http://www.jstor.org/stable/1266577
|
||||
// Knuth: The Art of Computer Programming Vol 2, section 4.2.2
|
||||
averageDiff += (diff - averageDiff) / ++count;
|
||||
}
|
||||
}
|
||||
aData += aBytesPerRow;
|
||||
bData += bBytesPerRow;
|
||||
}
|
||||
if (averageDiff > maxAvgDiff) equal = NO;
|
||||
|
||||
err = CVPixelBufferUnlockBaseAddress(b, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(err, kCVReturnSuccess);
|
||||
err = CVPixelBufferUnlockBaseAddress(a, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(err, kCVReturnSuccess);
|
||||
return equal;
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)convertPixelBuffer:(CVPixelBufferRef)input
|
||||
toPixelFormat:(OSType)pixelFormat {
|
||||
size_t width = CVPixelBufferGetWidth(input);
|
||||
size_t height = CVPixelBufferGetHeight(input);
|
||||
CVPixelBufferRef output;
|
||||
CVReturn status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault, width, height, pixelFormat,
|
||||
GetCVPixelBufferAttributesForGlCompatibility(), &output);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = vImageConvertCVPixelBuffers(input, output);
|
||||
XCTAssertEqual(status, kvImageNoError);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)scaleBGRAPixelBuffer:(CVPixelBufferRef)input
|
||||
toSize:(CGSize)size {
|
||||
CVPixelBufferRef output;
|
||||
CVReturn status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault, size.width, size.height, kCVPixelFormatType_32BGRA,
|
||||
GetCVPixelBufferAttributesForGlCompatibility(), &output);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
vImage_Buffer src = vImageForCVPixelBuffer(input);
|
||||
vImage_Buffer dst = vImageForCVPixelBuffer(output);
|
||||
status = vImageScale_ARGB8888(&src, &dst, NULL, kvImageNoFlags);
|
||||
XCTAssertEqual(status, kvImageNoError);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
- (CVPixelBufferRef)transformPixelBuffer:(CVPixelBufferRef)input
|
||||
outputPixelFormat:(OSType)pixelFormat
|
||||
transformation:(void(^)(CVPixelBufferRef input,
|
||||
CVPixelBufferRef output))transformation {
|
||||
size_t width = CVPixelBufferGetWidth(input);
|
||||
size_t height = CVPixelBufferGetHeight(input);
|
||||
CVPixelBufferRef output;
|
||||
CVReturn status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault, width, height,
|
||||
pixelFormat, NULL, &output);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
transformation(input, output);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(output, 0);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
status = CVPixelBufferUnlockBaseAddress(input, kCVPixelBufferLock_ReadOnly);
|
||||
XCTAssertEqual(status, kCVReturnSuccess);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
- (UIImage*)differenceOfImage:(UIImage*)inputA image:(UIImage*)inputB {
|
||||
UIGraphicsBeginImageContextWithOptions(inputA.size, YES, 1.0);
|
||||
CGRect imageRect = CGRectMake(0, 0, inputA.size.width, inputA.size.height);
|
||||
|
||||
[inputA drawInRect:imageRect blendMode:kCGBlendModeNormal alpha:1.0];
|
||||
[inputB drawInRect:imageRect blendMode:kCGBlendModeDifference alpha:1.0];
|
||||
|
||||
UIImage *differenceImage = UIGraphicsGetImageFromCurrentImageContext();
|
||||
UIGraphicsEndImageContext();
|
||||
|
||||
return differenceImage;
|
||||
}
|
||||
|
||||
- (void)testGraph:(MPPGraph*)graph
|
||||
input:(CVPixelBufferRef)inputBuffer
|
||||
expectedOutput:(CVPixelBufferRef)expectedBuffer
|
||||
{
|
||||
CVPixelBufferRef outputBuffer = [self runGraph:graph
|
||||
withPixelBuffer:inputBuffer
|
||||
packetType:MediaPipePacketPixelBuffer];
|
||||
#if DEBUG
|
||||
// Xcode can display UIImage objects right in the debugger. It is handy to
|
||||
// have these variables defined if the test fails.
|
||||
UIImage* output = UIImageWithPixelBuffer(outputBuffer);
|
||||
XCTAssertNotNil(output);
|
||||
UIImage* expected = UIImageWithPixelBuffer(expectedBuffer);
|
||||
XCTAssertNotNil(expected);
|
||||
UIImage* diff = [self differenceOfImage:output image:expected];
|
||||
(void)diff; // Suppress unused variable warning.
|
||||
#endif
|
||||
XCTAssert([self pixelBuffer:outputBuffer isCloseTo:expectedBuffer
|
||||
maxLocalDifference:INT_MAX maxAverageDifference:1]);
|
||||
CFRelease(outputBuffer);
|
||||
}
|
||||
|
||||
- (void)testGraphConfig:(const mediapipe::CalculatorGraphConfig&)config
|
||||
inputStreamsAndFiles:(NSDictionary<NSString*, NSString*>*)inputs
|
||||
outputStream:(NSString*)outputStream
|
||||
expectedOutputFile:(NSString*)expectedPath {
|
||||
[self testGraphConfig:config
|
||||
inputStreamsAndFiles:inputs
|
||||
inputStreamsAndPackets:{}
|
||||
sidePackets:{}
|
||||
timestamp:mediapipe::Timestamp(1)
|
||||
outputStream:outputStream
|
||||
expectedOutputFile:expectedPath
|
||||
maxAverageDifference:1.f];
|
||||
}
|
||||
|
||||
- (void)testGraphConfig:(const mediapipe::CalculatorGraphConfig&)config
|
||||
inputStreamsAndFiles:(NSDictionary<NSString*, NSString*>*)fileInputs
|
||||
inputStreamsAndPackets:(const std::map<std::string, mediapipe::Packet>&)packetInputs
|
||||
sidePackets:(std::map<std::string, mediapipe::Packet>)sidePackets
|
||||
timestamp:(mediapipe::Timestamp)timestamp
|
||||
outputStream:(NSString*)outputStream
|
||||
expectedOutputFile:(NSString*)expectedPath
|
||||
maxAverageDifference:(float)maxAverageDifference {
|
||||
NSBundle* testBundle = [NSBundle bundleForClass:[self class]];
|
||||
chdir([testBundle.resourcePath fileSystemRepresentation]);
|
||||
MPPGraph* graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[graph addSidePackets:sidePackets];
|
||||
[graph addFrameOutputStream:outputStream.UTF8String
|
||||
outputPacketType:MediaPipePacketPixelBuffer];
|
||||
|
||||
std::unordered_map<std::string, CFHolder<CVPixelBufferRef>> inputBuffers;
|
||||
for (NSString* inputStream in fileInputs) {
|
||||
UIImage* inputImage = [self testImageNamed:fileInputs[inputStream] extension:nil];
|
||||
XCTAssertNotNil(inputImage);
|
||||
::mediapipe::Status status =
|
||||
CreateCVPixelBufferFromCGImage(inputImage.CGImage, &inputBuffers[inputStream.UTF8String]);
|
||||
XCTAssert(status.ok());
|
||||
}
|
||||
|
||||
UIImage* expectedImage = [self testImageNamed:expectedPath extension:nil];
|
||||
XCTAssertNotNil(expectedImage);
|
||||
CFHolder<CVPixelBufferRef> expectedBuffer;
|
||||
::mediapipe::Status status =
|
||||
CreateCVPixelBufferFromCGImage(expectedImage.CGImage, &expectedBuffer);
|
||||
XCTAssert(status.ok());
|
||||
|
||||
CVPixelBufferRef outputBuffer = [self runGraph:graph
|
||||
withInputPixelBuffers:inputBuffers
|
||||
inputPackets:packetInputs
|
||||
timestamp:timestamp
|
||||
outputStream:outputStream.UTF8String
|
||||
packetType:MediaPipePacketPixelBuffer];
|
||||
|
||||
UIImage* output = UIImageWithPixelBuffer(outputBuffer);
|
||||
XCTAssertNotNil(output);
|
||||
|
||||
UIImage* expected = UIImageWithPixelBuffer(*expectedBuffer);
|
||||
XCTAssertNotNil(expected);
|
||||
UIImage* diff = [self differenceOfImage:output image:expected];
|
||||
|
||||
XCTAssert([self pixelBuffer:outputBuffer isCloseTo:*expectedBuffer
|
||||
maxLocalDifference:INT_MAX maxAverageDifference:maxAverageDifference]);
|
||||
|
||||
CFRelease(outputBuffer);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,342 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <XCTest/XCTest.h>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#import "mediapipe/objc/MPPGraph.h"
|
||||
#import "mediapipe/objc/MPPGraphTestBase.h"
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
#import "mediapipe/objc/Weakify.h"
|
||||
#import "mediapipe/objc/util.h"
|
||||
|
||||
static const char* kExpectedError = "Expected error.";
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class GrayscaleCalculator : public Calculator {
|
||||
public:
|
||||
static ::mediapipe::Status FillExpectations(const CalculatorOptions& options,
|
||||
PacketTypeSet* inputs, PacketTypeSet* outputs,
|
||||
PacketTypeSet* input_side_packets) {
|
||||
inputs->Index(0).Set<ImageFrame>();
|
||||
outputs->Index(0).Set<ImageFrame>();
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process() final {
|
||||
const auto& input = Input()->Get<ImageFrame>();
|
||||
int w = input.Width();
|
||||
int h = input.Height();
|
||||
|
||||
auto output = absl::make_unique<mediapipe::ImageFrame>(ImageFormat::GRAY8, w, h);
|
||||
|
||||
vImage_Buffer src = vImageForImageFrame(input);
|
||||
vImage_Buffer dst = vImageForImageFrame(*output);
|
||||
vImage_Error vErr = vImageRGBAToGray(&src, &dst);
|
||||
NSCAssert(vErr == kvImageNoError, @"vImageRGBAToGray failed: %zd", vErr);
|
||||
|
||||
Output()->Add(output.release(), InputTimestamp());
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(GrayscaleCalculator);
|
||||
|
||||
// For testing that video header is present. Open() will have a failure status
|
||||
// if the video header is not present in the input stream.
|
||||
class VideoHeaderCalculator : public Calculator {
|
||||
public:
|
||||
static ::mediapipe::Status FillExpectations(const CalculatorOptions& options,
|
||||
PacketTypeSet* inputs, PacketTypeSet* outputs,
|
||||
PacketTypeSet* input_side_packets) {
|
||||
inputs->Index(0).Set<ImageFrame>();
|
||||
outputs->Index(0).Set<ImageFrame>();
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open() final {
|
||||
if (Input()->Header().IsEmpty()) {
|
||||
return ::util::UnknownError("No video header present.");
|
||||
}
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process() final {
|
||||
Output()->AddPacket(Input()->Value());
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(VideoHeaderCalculator);
|
||||
|
||||
class ErrorCalculator : public Calculator {
|
||||
public:
|
||||
static ::mediapipe::Status FillExpectations(const CalculatorOptions& options,
|
||||
PacketTypeSet* inputs, PacketTypeSet* outputs,
|
||||
PacketTypeSet* input_side_packets) {
|
||||
inputs->Index(0).SetAny();
|
||||
outputs->Index(0).SetSameAs(&inputs->Index(0));
|
||||
return ::util::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process() final {
|
||||
return ::mediapipe::Status(absl::StatusCode::kUnknown, kExpectedError);
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(ErrorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@interface MPPGraphTests : MPPGraphTestBase{
|
||||
UIImage* _sourceImage;
|
||||
MPPGraph* _graph;
|
||||
}
|
||||
@end
|
||||
|
||||
@implementation MPPGraphTests
|
||||
|
||||
- (void)setUp {
|
||||
[super setUp];
|
||||
|
||||
_sourceImage = [self testImageNamed:@"googlelogo_color_272x92dp" extension:@"png"];
|
||||
}
|
||||
|
||||
- (void)tearDown {
|
||||
[super tearDown];
|
||||
}
|
||||
|
||||
- (void)testPassThrough {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("PassThroughCalculator");
|
||||
node->add_input_stream("input_frames");
|
||||
node->add_output_stream("output_frames");
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_frames" outputPacketType:MediaPipePacketPixelBuffer];
|
||||
CFHolder<CVPixelBufferRef> inputBuffer;
|
||||
::mediapipe::Status status = CreateCVPixelBufferFromCGImage(_sourceImage.CGImage, &inputBuffer);
|
||||
XCTAssert(status.ok());
|
||||
CVPixelBufferRef outputBuffer = [self runGraph:_graph
|
||||
withPixelBuffer:*inputBuffer
|
||||
packetType:MediaPipePacketPixelBuffer];
|
||||
XCTAssert([self pixelBuffer:outputBuffer isEqualTo:*inputBuffer]);
|
||||
}
|
||||
|
||||
- (UIImage*)grayImage:(UIImage*)inputImage {
|
||||
UIGraphicsBeginImageContextWithOptions(inputImage.size, YES, 1.0);
|
||||
CGRect imageRect = CGRectMake(0, 0, inputImage.size.width, inputImage.size.height);
|
||||
|
||||
// Draw the image with the luminosity blend mode.
|
||||
// On top of a white background, this will give a black and white image.
|
||||
[inputImage drawInRect:imageRect blendMode:kCGBlendModeLuminosity alpha:1.0];
|
||||
|
||||
UIImage *filteredImage = UIGraphicsGetImageFromCurrentImageContext();
|
||||
UIGraphicsEndImageContext();
|
||||
|
||||
return filteredImage;
|
||||
}
|
||||
|
||||
- (void)testMultipleOutputs {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto passThroughNode = config.add_node();
|
||||
passThroughNode->set_calculator("PassThroughCalculator");
|
||||
passThroughNode->add_input_stream("input_frames");
|
||||
passThroughNode->add_output_stream("pass_frames");
|
||||
auto grayNode = config.add_node();
|
||||
grayNode->set_calculator("GrayscaleCalculator");
|
||||
grayNode->add_input_stream("input_frames");
|
||||
grayNode->add_output_stream("gray_frames");
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"pass_frames" outputPacketType:MediaPipePacketImageFrame];
|
||||
[_graph addFrameOutputStream:"gray_frames" outputPacketType:MediaPipePacketImageFrame];
|
||||
|
||||
CFHolder<CVPixelBufferRef> inputBuffer;
|
||||
::mediapipe::Status status = CreateCVPixelBufferFromCGImage(_sourceImage.CGImage, &inputBuffer);
|
||||
XCTAssert(status.ok());
|
||||
|
||||
WEAKIFY(self);
|
||||
XCTestExpectation* passFrameReceive =
|
||||
[self expectationWithDescription:@"pass through output received"];
|
||||
XCTestExpectation* grayFrameReceive =
|
||||
[self expectationWithDescription:@"grayscale output received"];
|
||||
_pixelBufferOutputBlock = ^(MPPGraph* outputGraph, CVPixelBufferRef outputBuffer,
|
||||
const std::string& outputStreamName) {
|
||||
STRONGIFY(self);
|
||||
XCTAssertEqualObjects(outputGraph, self->_graph);
|
||||
if (outputStreamName == "pass_frames") {
|
||||
[passFrameReceive fulfill];
|
||||
} else if (outputStreamName == "gray_frames") {
|
||||
[grayFrameReceive fulfill];
|
||||
}
|
||||
};
|
||||
|
||||
[self runGraph:_graph withPixelBuffer:*inputBuffer packetType:MediaPipePacketImageFrame];
|
||||
}
|
||||
|
||||
- (void)testGrayscaleOutput {
|
||||
// When a calculator outputs a grayscale ImageFrame, it is returned to the
|
||||
// application as a BGRA pixel buffer. To test it, let's feed a grayscale
|
||||
// image into the graph and make sure it comes out unscathed.
|
||||
UIImage* grayImage = [self grayImage:_sourceImage];
|
||||
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("GrayscaleCalculator");
|
||||
node->add_input_stream("input_frames");
|
||||
node->add_output_stream("output_frames");
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_frames" outputPacketType:MediaPipePacketImageFrame];
|
||||
CFHolder<CVPixelBufferRef> inputBuffer;
|
||||
::mediapipe::Status status = CreateCVPixelBufferFromCGImage(grayImage.CGImage, &inputBuffer);
|
||||
XCTAssert(status.ok());
|
||||
CVPixelBufferRef outputBuffer = [self runGraph:_graph
|
||||
withPixelBuffer:*inputBuffer
|
||||
packetType:MediaPipePacketImageFrame];
|
||||
// We accept a small difference due to gamma correction and whatnot.
|
||||
XCTAssert([self pixelBuffer:outputBuffer isCloseTo:*inputBuffer
|
||||
maxLocalDifference:5 maxAverageDifference:FLT_MAX]);
|
||||
}
|
||||
|
||||
- (void)testGraphError {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("ErrorCalculator");
|
||||
node->add_input_stream("input_frames");
|
||||
node->add_output_stream("output_frames");
|
||||
CFHolder<CVPixelBufferRef> srcPixelBuffer;
|
||||
::mediapipe::Status status =
|
||||
CreateCVPixelBufferFromCGImage(_sourceImage.CGImage, &srcPixelBuffer);
|
||||
XCTAssert(status.ok());
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_frames" outputPacketType:MediaPipePacketImageFrame];
|
||||
_graph.delegate = self;
|
||||
|
||||
XCTAssert([_graph startWithError:nil]);
|
||||
[_graph sendPixelBuffer:*srcPixelBuffer
|
||||
intoStream:"input_frames"
|
||||
packetType:MediaPipePacketImageFrame];
|
||||
XCTAssert([_graph closeInputStream:"input_frames" error:nil]);
|
||||
|
||||
__block NSError* error = nil;
|
||||
XCTestExpectation* graphDone = [self expectationWithDescription:@"graph done"];
|
||||
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
|
||||
XCTAssertFalse([_graph waitUntilDoneWithError:&error]);
|
||||
[graphDone fulfill];
|
||||
});
|
||||
|
||||
[self waitForExpectationsWithTimeout:3.0 handler:NULL];
|
||||
XCTAssertNotNil(error);
|
||||
status = error.gus_status;
|
||||
XCTAssertNotEqual(status.error_message().find(kExpectedError), std::string::npos,
|
||||
@"Missing expected std::string '%s' from error messge '%s'", kExpectedError,
|
||||
status.error_message().c_str());
|
||||
}
|
||||
|
||||
- (void)testSetStreamHeader {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("VideoHeaderCalculator");
|
||||
node->add_input_stream("input_frames");
|
||||
node->add_output_stream("output_frames");
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_frames" outputPacketType:MediaPipePacketImageFrame];
|
||||
|
||||
// We're no longer using video headers, let's just use an int as the header.
|
||||
auto header_packet = mediapipe::MakePacket<int>(0xDEADBEEF);
|
||||
[_graph setHeaderPacket:header_packet forStream:"input_frames"];
|
||||
|
||||
// Verify that Open() on calculator succeeded.
|
||||
XCTAssert([_graph startWithError:nil]);
|
||||
|
||||
// Tear down graph.
|
||||
XCTAssert([_graph closeInputStream:"input_frames" error:nil]);
|
||||
XCTestExpectation* graphDone = [self expectationWithDescription:@"graph done"];
|
||||
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
|
||||
XCTAssert([_graph waitUntilDoneWithError:nil]);
|
||||
[graphDone fulfill];
|
||||
});
|
||||
|
||||
[self waitForExpectationsWithTimeout:3.0 handler:NULL];
|
||||
}
|
||||
|
||||
- (void)testGraphIsDeallocated {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_frames");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("PassThroughCalculator");
|
||||
node->add_input_stream("input_frames");
|
||||
node->add_output_stream("output_frames");
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_frames" outputPacketType:MediaPipePacketPixelBuffer];
|
||||
CFHolder<CVPixelBufferRef> inputBuffer;
|
||||
::mediapipe::Status status = CreateCVPixelBufferFromCGImage(_sourceImage.CGImage, &inputBuffer);
|
||||
XCTAssert(status.ok());
|
||||
CVPixelBufferRef outputBuffer = [self runGraph:_graph
|
||||
withPixelBuffer:*inputBuffer
|
||||
packetType:MediaPipePacketPixelBuffer];
|
||||
__weak MPPGraph* weakGraph = _graph;
|
||||
_graph = nil;
|
||||
XCTAssertNil(weakGraph);
|
||||
}
|
||||
|
||||
- (void)testRawPacketOutput {
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input_ints");
|
||||
auto node = config.add_node();
|
||||
node->set_calculator("PassThroughCalculator");
|
||||
node->add_input_stream("input_ints");
|
||||
node->add_output_stream("output_ints");
|
||||
|
||||
const int kTestValue = 10;
|
||||
|
||||
_graph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[_graph addFrameOutputStream:"output_ints" outputPacketType:MediaPipePacketRaw];
|
||||
_graph.delegate = self;
|
||||
|
||||
WEAKIFY(self);
|
||||
XCTestExpectation* outputReceived = [self expectationWithDescription:@"output received"];
|
||||
_packetOutputBlock = ^(MPPGraph* outputGraph, const mediapipe::Packet& packet,
|
||||
const std::string& outputStreamName) {
|
||||
STRONGIFY(self);
|
||||
XCTAssertEqualObjects(outputGraph, _graph);
|
||||
XCTAssertEqual(outputStreamName, "output_ints");
|
||||
XCTAssertEqual(packet.Get<int>(), kTestValue);
|
||||
[outputReceived fulfill];
|
||||
};
|
||||
|
||||
XCTAssert([_graph startWithError:nil]);
|
||||
XCTAssert([_graph sendPacket:mediapipe::MakePacket<int>(kTestValue).At(mediapipe::Timestamp(1))
|
||||
intoStream:"input_ints"
|
||||
error:nil]);
|
||||
XCTAssert([_graph closeInputStream:"input_ints" error:nil]);
|
||||
XCTestExpectation* graphDone = [self expectationWithDescription:@"graph done"];
|
||||
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
|
||||
XCTAssert([_graph waitUntilDoneWithError:nil]);
|
||||
[graphDone fulfill];
|
||||
});
|
||||
|
||||
[self waitForExpectationsWithTimeout:3.0 handler:NULL];
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
@class MPPInputSource;
|
||||
|
||||
/// A delegate that can receive frames from a source.
|
||||
@protocol MPPInputSourceDelegate <NSObject>
|
||||
|
||||
/// Provides the delegate with a new video frame.
|
||||
@optional
|
||||
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
|
||||
fromSource:(MPPInputSource*)source __deprecated;
|
||||
|
||||
/// Provides the delegate with a new video frame.
|
||||
@optional
|
||||
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
|
||||
timestamp:(CMTime)timestamp
|
||||
fromSource:(MPPInputSource*)source;
|
||||
|
||||
// Provides the delegate with a new depth frame data
|
||||
@optional
|
||||
- (void)processDepthData:(AVDepthData*)depthData
|
||||
timestamp:(CMTime)timestamp
|
||||
fromSource:(MPPInputSource*)source;
|
||||
|
||||
@optional
|
||||
- (void)videoDidPlayToEnd:(CMTime)timestamp;
|
||||
|
||||
@end
|
||||
|
||||
/// Abstract class for a video source.
|
||||
@interface MPPInputSource : NSObject
|
||||
|
||||
/// The delegate that receives the frames.
|
||||
@property(weak, nonatomic, readonly) id<MPPInputSourceDelegate> delegate;
|
||||
|
||||
/// The dispatch queue on which to schedule the delegate callback.
|
||||
@property(nonatomic, readonly) dispatch_queue_t delegateQueue;
|
||||
|
||||
/// Whether the source is currently running.
|
||||
@property(nonatomic, getter=isRunning, readonly) BOOL running;
|
||||
|
||||
/// Sets the delegate and the queue on which its callback should be invoked.
|
||||
- (void)setDelegate:(id<MPPInputSourceDelegate>)delegate queue:(dispatch_queue_t)queue;
|
||||
|
||||
/// CoreVideo pixel format for the video frames. Defaults to
|
||||
/// kCVPixelFormatType_32BGRA.
|
||||
@property(nonatomic) OSType pixelFormatType;
|
||||
|
||||
/// Starts the source.
|
||||
- (void)start;
|
||||
|
||||
/// Stops the source.
|
||||
- (void)stop;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPInputSource.h"
|
||||
|
||||
#define ABSTRACT_METHOD \
|
||||
@throw [NSException exceptionWithName:NSInternalInconsistencyException \
|
||||
reason:[NSString stringWithFormat:@"You must override %@ in a subclass", \
|
||||
NSStringFromSelector(_cmd)] \
|
||||
userInfo:nil];
|
||||
|
||||
@implementation MPPInputSource
|
||||
|
||||
- (void)setDelegate:(id<MPPInputSourceDelegate>)delegate
|
||||
queue:(dispatch_queue_t)queue {
|
||||
_delegate = delegate;
|
||||
_delegateQueue = queue;
|
||||
}
|
||||
|
||||
- (void)start {
|
||||
ABSTRACT_METHOD
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
ABSTRACT_METHOD
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <GLKit/GLKit.h>
|
||||
|
||||
#import "mediapipe/objc/MPPGLViewRenderer.h"
|
||||
|
||||
/// Renders frames in a Core Animation layer.
|
||||
@interface MPPLayerRenderer : NSObject
|
||||
|
||||
@property(nonatomic, readonly) CAEAGLLayer *layer;
|
||||
|
||||
/// Updates the layer with a new pixel buffer.
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer;
|
||||
|
||||
/// Sets which way to rotate input frames before rendering them.
|
||||
/// Default value is MediaPipeFrameRotationNone.
|
||||
@property(nonatomic) MediaPipeFrameRotationMode frameRotationMode;
|
||||
|
||||
/// Sets how to scale the frame within the layer.
|
||||
/// Default value is MediaPipeFrameScaleScaleToFit.
|
||||
@property(nonatomic) MediaPipeFrameScaleMode frameScaleMode;
|
||||
|
||||
/// If YES, swap left and right. Useful for the front camera.
|
||||
@property(nonatomic) BOOL mirrored;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPLayerRenderer.h"
|
||||
|
||||
@implementation MPPLayerRenderer {
|
||||
/// Used for rendering to a CAEAGLLayer.
|
||||
MPPGLViewRenderer *_glRenderer;
|
||||
|
||||
/// Used for rendering.
|
||||
GLuint framebuffer_;
|
||||
GLuint renderbuffer_;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_layer = [[CAEAGLLayer alloc] init];
|
||||
// The default drawable properties are ok.
|
||||
_layer.opaque = YES;
|
||||
// Synchronizes presentation with CoreAnimation transactions.
|
||||
// Avoids desync between GL and CA updates.
|
||||
// TODO: should this be on by default? It's not on in GLKView,
|
||||
// but if we add support for AVSampleBufferDisplayLayer we may want to
|
||||
// make sure the behavior is similar.
|
||||
if ([_layer respondsToSelector:@selector(setPresentsWithTransaction:)]) {
|
||||
_layer.presentsWithTransaction = YES;
|
||||
}
|
||||
_layer.contentsScale = [[UIScreen mainScreen] scale];
|
||||
_glRenderer = [[MPPGLViewRenderer alloc] init];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
}
|
||||
|
||||
- (void)setupFrameBuffer {
|
||||
[EAGLContext setCurrentContext:_glRenderer.glContext];
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
glGenFramebuffers(1, &framebuffer_);
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
|
||||
glGenRenderbuffers(1, &renderbuffer_);
|
||||
glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer_);
|
||||
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, renderbuffer_);
|
||||
[_glRenderer.glContext renderbufferStorage:GL_RENDERBUFFER fromDrawable:_layer];
|
||||
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
|
||||
NSAssert(status == GL_FRAMEBUFFER_COMPLETE,
|
||||
@"failed to make complete framebuffer object %x", status);
|
||||
}
|
||||
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer {
|
||||
if (!framebuffer_) {
|
||||
[self setupFrameBuffer];
|
||||
}
|
||||
GLfloat drawWidth = _layer.bounds.size.width * _layer.contentsScale;
|
||||
GLfloat drawHeight = _layer.bounds.size.height * _layer.contentsScale;
|
||||
[EAGLContext setCurrentContext:_glRenderer.glContext];
|
||||
glViewport(0, 0, drawWidth, drawHeight);
|
||||
[_glRenderer drawPixelBuffer:pixelBuffer width:drawWidth height:drawHeight];
|
||||
BOOL success = [_glRenderer.glContext presentRenderbuffer:GL_RENDERBUFFER];
|
||||
if (!success) NSLog(@"presentRenderbuffer failed");
|
||||
}
|
||||
|
||||
- (MediaPipeFrameRotationMode)frameRotationMode {
|
||||
return _glRenderer.frameRotationMode;
|
||||
}
|
||||
|
||||
- (void)setFrameRotationMode:(MediaPipeFrameRotationMode)frameRotationMode {
|
||||
_glRenderer.frameRotationMode = frameRotationMode;
|
||||
}
|
||||
|
||||
- (MediaPipeFrameScaleMode)frameScaleMode {
|
||||
return _glRenderer.frameScaleMode;
|
||||
}
|
||||
|
||||
- (void)setFrameScaleMode:(MediaPipeFrameScaleMode)frameScaleMode {
|
||||
_glRenderer.frameScaleMode = frameScaleMode;
|
||||
}
|
||||
|
||||
- (BOOL)mirrored {
|
||||
return _glRenderer.mirrored;
|
||||
}
|
||||
|
||||
- (void)setMirrored:(BOOL)mirrored {
|
||||
_glRenderer.mirrored = mirrored;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPInputSource.h"
|
||||
|
||||
/// A source that outputs frames from a video, played in real time.
|
||||
/// Not meant for batch processing of video.
|
||||
@interface MPPPlayerInputSource : MPPInputSource
|
||||
|
||||
/// Designated initializer.
|
||||
- (instancetype)initWithAVAsset:(AVAsset*)video;
|
||||
|
||||
/// Skip into video @c time from beginning (time 0), within error of +/- tolerance to closest time.
|
||||
- (void)seekToTime:(CMTime)time tolerance:(CMTime)tolerance;
|
||||
|
||||
/// Set time into video at which to end playback.
|
||||
- (void)setPlaybackEndTime:(CMTime)time;
|
||||
|
||||
/// Returns the current video's timestamp.
|
||||
- (CMTime)currentPlayerTime;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPPlayerInputSource.h"
|
||||
#import "mediapipe/objc/MPPDisplayLinkWeakTarget.h"
|
||||
|
||||
@implementation MPPPlayerInputSource {
|
||||
AVAsset* _video;
|
||||
AVPlayerItem* _videoItem;
|
||||
AVPlayer* _videoPlayer;
|
||||
AVPlayerItemVideoOutput* _videoOutput;
|
||||
CADisplayLink* _videoDisplayLink;
|
||||
MPPDisplayLinkWeakTarget* _displayLinkWeakTarget;
|
||||
id _videoEndObserver;
|
||||
}
|
||||
|
||||
- (instancetype)initWithAVAsset:(AVAsset*)video {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_video = video;
|
||||
_videoItem = [AVPlayerItem playerItemWithAsset:_video];
|
||||
// Necessary to ensure the video's preferred transform is respected.
|
||||
_videoItem.videoComposition = [AVVideoComposition videoCompositionWithPropertiesOfAsset:_video];
|
||||
|
||||
_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:@{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
|
||||
(id)kCVPixelBufferIOSurfacePropertiesKey : [NSDictionary dictionary]
|
||||
}];
|
||||
_videoOutput.suppressesPlayerRendering = YES;
|
||||
[_videoItem addOutput:_videoOutput];
|
||||
|
||||
_displayLinkWeakTarget =
|
||||
[[MPPDisplayLinkWeakTarget alloc] initWithTarget:self selector:@selector(videoUpdate:)];
|
||||
|
||||
_videoDisplayLink = [CADisplayLink displayLinkWithTarget:_displayLinkWeakTarget
|
||||
selector:@selector(displayLinkCallback:)];
|
||||
_videoDisplayLink.paused = YES;
|
||||
[_videoDisplayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
|
||||
|
||||
_videoPlayer = [AVPlayer playerWithPlayerItem:_videoItem];
|
||||
_videoPlayer.actionAtItemEnd = AVPlayerActionAtItemEndNone;
|
||||
NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
|
||||
_videoEndObserver = [center addObserverForName:AVPlayerItemDidPlayToEndTimeNotification
|
||||
object:_videoItem
|
||||
queue:nil
|
||||
usingBlock:^(NSNotification* note) {
|
||||
[self playerItemDidPlayToEnd:note];
|
||||
}];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)start {
|
||||
[_videoPlayer play];
|
||||
_videoDisplayLink.paused = NO;
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
_videoDisplayLink.paused = YES;
|
||||
[_videoPlayer pause];
|
||||
}
|
||||
|
||||
- (BOOL)isRunning {
|
||||
return _videoPlayer.rate != 0.0;
|
||||
}
|
||||
|
||||
- (void)videoUpdate:(CADisplayLink*)sender {
|
||||
CMTime timestamp = [_videoItem currentTime];
|
||||
if ([_videoOutput hasNewPixelBufferForItemTime:timestamp]) {
|
||||
CVPixelBufferRef pixelBuffer =
|
||||
[_videoOutput copyPixelBufferForItemTime:timestamp itemTimeForDisplay:nil];
|
||||
if (pixelBuffer)
|
||||
dispatch_async(self.delegateQueue, ^{
|
||||
if ([self.delegate respondsToSelector:@selector(processVideoFrame:timestamp:fromSource:)]) {
|
||||
[self.delegate processVideoFrame:pixelBuffer timestamp:timestamp fromSource:self];
|
||||
} else if ([self.delegate respondsToSelector:@selector(processVideoFrame:fromSource:)]) {
|
||||
[self.delegate processVideoFrame:pixelBuffer fromSource:self];
|
||||
}
|
||||
CFRelease(pixelBuffer);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[[NSNotificationCenter defaultCenter] removeObserver:self];
|
||||
[_videoDisplayLink invalidate];
|
||||
_videoPlayer = nil;
|
||||
}
|
||||
|
||||
#pragma mark - NSNotificationCenter / observer
|
||||
|
||||
- (void)playerItemDidPlayToEnd:(NSNotification*)notification {
|
||||
CMTime timestamp = [_videoItem currentTime];
|
||||
dispatch_async(self.delegateQueue, ^{
|
||||
if ([self.delegate respondsToSelector:@selector(videoDidPlayToEnd:)]) {
|
||||
[self.delegate videoDidPlayToEnd:timestamp];
|
||||
} else {
|
||||
// Default to loop if no delegate handler set.
|
||||
[_videoPlayer seekToTime:kCMTimeZero];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
- (void)seekToTime:(CMTime)time tolerance:(CMTime)tolerance {
|
||||
[_videoPlayer seekToTime:time toleranceBefore:tolerance toleranceAfter:tolerance];
|
||||
}
|
||||
|
||||
- (void)setPlaybackEndTime:(CMTime)time {
|
||||
_videoPlayer.currentItem.forwardPlaybackEndTime = time;
|
||||
}
|
||||
|
||||
- (CMTime)currentPlayerTime {
|
||||
return _videoPlayer.currentTime;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/objc/util.h"
|
||||
|
||||
/// Helps convert a CMTime to a MediaPipe timestamp.
|
||||
@interface MPPTimestampConverter : NSObject
|
||||
|
||||
/// The last timestamp returned by timestampForMediaTime:.
|
||||
@property(nonatomic, readonly) mediapipe::Timestamp lastTimestamp;
|
||||
|
||||
/// Initializer.
|
||||
- (instancetype)init NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/// Resets the object. After this method is called, we can return timestamps
|
||||
/// that are lower than previously returned timestamps.
|
||||
- (void)reset;
|
||||
|
||||
/// Converts a CMTime to a MediaPipe timestamp. This ensures that MediaPipe
|
||||
/// timestamps
|
||||
/// are always increasing: if the provided CMTime has gone backwards (e.g. if
|
||||
/// it's from a
|
||||
/// looping video), we shift all timestamps from that point on to keep the
|
||||
/// output increasing.
|
||||
/// This state is erased when reset is called.
|
||||
- (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "MPPTimestampConverter.h"
|
||||
|
||||
@implementation MPPTimestampConverter {
|
||||
mediapipe::Timestamp _mediapipeTimestamp;
|
||||
mediapipe::Timestamp _lastTimestamp;
|
||||
mediapipe::TimestampDiff _timestampOffset;
|
||||
}
|
||||
|
||||
- (instancetype)init
|
||||
{
|
||||
self = [super init];
|
||||
if (self) {
|
||||
[self reset];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)reset {
|
||||
_mediapipeTimestamp = mediapipe::Timestamp::Min();
|
||||
_lastTimestamp = _mediapipeTimestamp;
|
||||
_timestampOffset = 0;
|
||||
}
|
||||
|
||||
- (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime {
|
||||
float sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0;
|
||||
const int64 sampleUsec = sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond;
|
||||
_mediapipeTimestamp = mediapipe::Timestamp(sampleUsec) + _timestampOffset;
|
||||
if (_mediapipeTimestamp <= _lastTimestamp) {
|
||||
_timestampOffset = _timestampOffset + _lastTimestamp + 1 - _mediapipeTimestamp;
|
||||
_mediapipeTimestamp = _lastTimestamp + 1;
|
||||
}
|
||||
_lastTimestamp = _mediapipeTimestamp;
|
||||
return _mediapipeTimestamp;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "Foundation/Foundation.h"
|
||||
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
/// Error domain for ::mediapipe::Status errors.
|
||||
extern NSString *const kGUSGoogleUtilStatusErrorDomain;
|
||||
|
||||
/// Key for the ::mediapipe::Status wrapper in an NSError's user info dictionary.
|
||||
extern NSString *const kGUSGoogleUtilStatusErrorKey;
|
||||
|
||||
/// This just wraps ::mediapipe::Status into an Objective-C object.
|
||||
@interface GUSUtilStatusWrapper : NSObject
|
||||
|
||||
@property(nonatomic)::mediapipe::Status status;
|
||||
|
||||
+ (instancetype)wrapStatus:(const ::mediapipe::Status &)status;
|
||||
|
||||
@end
|
||||
|
||||
/// This category adds methods for generating NSError objects from ::mediapipe::Status
|
||||
/// objects, and vice versa.
|
||||
@interface NSError (GUSGoogleUtilStatus)
|
||||
|
||||
/// Generates an NSError representing a ::mediapipe::Status. Note that NSError always
|
||||
/// represents an error, so this should not be called with ::mediapipe::Status::OK.
|
||||
+ (NSError *)gus_errorWithStatus:(const ::mediapipe::Status &)status;
|
||||
|
||||
/// Returns a ::mediapipe::Status object representing an NSError. If the NSError was
|
||||
/// generated from a ::mediapipe::Status, the ::mediapipe::Status returned is identical to
|
||||
/// the original. Otherwise, this returns a status with code ::util::error::UNKNOWN
|
||||
/// and a message extracted from the NSError.
|
||||
@property(nonatomic, readonly)::mediapipe::Status gus_status; // NOLINT(identifier-naming)
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/objc/NSError+util_status.h"
|
||||
|
||||
@implementation GUSUtilStatusWrapper
|
||||
|
||||
+ (instancetype)wrapStatus:(const ::mediapipe::Status &)status {
|
||||
return [[self alloc] initWithStatus:status];
|
||||
}
|
||||
|
||||
- (instancetype)initWithStatus:(const ::mediapipe::Status &)status {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_status = status;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (NSString *)description {
|
||||
return [NSString stringWithFormat:@"<%@: %p; status = %s>",
|
||||
[self class], self, _status.error_message().c_str()];
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation NSError (GUSGoogleUtilStatus)
|
||||
|
||||
NSString *const kGUSGoogleUtilStatusErrorDomain = @"GoogleUtilStatusErrorDomain";
|
||||
NSString *const kGUSGoogleUtilStatusErrorKey = @"GUSGoogleUtilStatusErrorKey";
|
||||
|
||||
+ (NSError *)gus_errorWithStatus:(const ::mediapipe::Status &)status {
|
||||
NSDictionary *userInfo = @{
|
||||
NSLocalizedDescriptionKey : @(status.error_message().c_str()),
|
||||
kGUSGoogleUtilStatusErrorKey : [GUSUtilStatusWrapper wrapStatus:status],
|
||||
};
|
||||
NSError *error = [NSError errorWithDomain:kGUSGoogleUtilStatusErrorDomain
|
||||
code:static_cast<NSInteger>(status.code())
|
||||
userInfo:userInfo];
|
||||
return error;
|
||||
}
|
||||
|
||||
- (::mediapipe::Status)gus_status {
|
||||
NSString *domain = self.domain;
|
||||
if ([domain isEqual:kGUSGoogleUtilStatusErrorDomain]) {
|
||||
GUSUtilStatusWrapper *wrapper = self.userInfo[kGUSGoogleUtilStatusErrorKey];
|
||||
if (wrapper) return wrapper.status;
|
||||
#if 0
|
||||
// Unfortunately, util/task/posixerrorspace.h is not in portable status yet.
|
||||
// TODO: fix that.
|
||||
} else if ([domain isEqual:NSPOSIXErrorDomain]) {
|
||||
return ::util::PosixErrorToStatus(self.code, self.localizedDescription.UTF8String);
|
||||
#endif
|
||||
}
|
||||
return ::mediapipe::Status(mediapipe::StatusCode::kUnknown, self.localizedDescription.UTF8String);
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// OBJC_LINTER
|
||||
|
||||
// TODO: check license. This was forked from an internal header.
|
||||
|
||||
/**
|
||||
* __WEAKNAME_ is a private macro used to generate a local variable name related
|
||||
* to the argument variable name. This generated local variable name is
|
||||
* intentionally stable across multiple invocations.
|
||||
*/
|
||||
#define __WEAKNAME_(variable) variable##_weak_
|
||||
|
||||
/**
|
||||
* WEAKIFY defines a new local variable that is a weak reference to the argument
|
||||
* variable.
|
||||
*
|
||||
* This macro is generally used to capture a weak reference to be captured by an
|
||||
* Objective-C block to avoid unintentionally extending an object's lifetime or
|
||||
* avoid causing a retain cycle.
|
||||
*
|
||||
* The new local variable's name will be based on the name of the target
|
||||
* variable and is stable across multiple invocations of WEAKIFY. In general,
|
||||
* you should not need to invoke WEAKIFY multiple times on the same variable.
|
||||
*/
|
||||
#define WEAKIFY(variable) \
|
||||
__weak __typeof__(variable) __WEAKNAME_(variable) = (variable)
|
||||
|
||||
/**
|
||||
* STRONGIFY defines a new shadow local variable with the same name as the
|
||||
* argument variable and initialize it with a resolved weak reference based on a
|
||||
* weak reference created previously using the WEAKIFY macro.
|
||||
*
|
||||
* @note:
|
||||
* This macro must be called within each block scope to prevent nested blocks
|
||||
* from capturing a strong reference from an outer block.
|
||||
*/
|
||||
#define STRONGIFY(variable) \
|
||||
_Pragma("clang diagnostic push") \
|
||||
_Pragma("clang diagnostic ignored \"-Wshadow\"") \
|
||||
__strong __typeof__(variable) variable = \
|
||||
__WEAKNAME_(variable) _Pragma("clang diagnostic pop")
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 360 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 382 KiB |
+12
@@ -0,0 +1,12 @@
|
||||
input_stream: "input_video"
|
||||
|
||||
node: {
|
||||
calculator: "GlLuminanceCalculator"
|
||||
input_stream: "input_video"
|
||||
output_stream: "luma_video"
|
||||
}
|
||||
node: {
|
||||
calculator: "GlSobelCalculator"
|
||||
input_stream: "luma_video"
|
||||
output_stream: "output_video"
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/objc/util.h"
|
||||
|
||||
#include "absl/base/macros.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/source_location.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_builder.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// NOTE: you must release the colorspace returned by this function, unless
|
||||
// it's null.
|
||||
// Returns an invalid format (all fields 0) if the requested format is
|
||||
// unsupported.
|
||||
vImage_CGImageFormat vImageFormatForCVPixelFormat(OSType pixel_format) {
|
||||
switch (pixel_format) {
|
||||
case kCVPixelFormatType_OneComponent8:
|
||||
return {
|
||||
.bitsPerComponent = 8,
|
||||
.bitsPerPixel = 8,
|
||||
.colorSpace = CGColorSpaceCreateDeviceGray(),
|
||||
.bitmapInfo = kCGImageAlphaNone | kCGBitmapByteOrderDefault,
|
||||
};
|
||||
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
return {
|
||||
.bitsPerComponent = 8,
|
||||
.bitsPerPixel = 32,
|
||||
.colorSpace = NULL,
|
||||
.bitmapInfo = kCGImageAlphaFirst | kCGBitmapByteOrder32Little,
|
||||
};
|
||||
|
||||
case kCVPixelFormatType_32RGBA:
|
||||
return {
|
||||
.bitsPerComponent = 8,
|
||||
.bitsPerPixel = 32,
|
||||
.colorSpace = NULL,
|
||||
.bitmapInfo = kCGImageAlphaLast | kCGBitmapByteOrderDefault,
|
||||
};
|
||||
|
||||
default:
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
CGColorSpaceRef CreateConversionCGColorSpaceForPixelFormat(
|
||||
OSType pixel_format) {
|
||||
// According to vImage documentation, YUV formats require the RGB colorspace
|
||||
// in which the RGB conversion should be interpreted. sRGB is suggested.
|
||||
// We cannot just pass sRGB all the time, though, since it breaks with
|
||||
// monochrome.
|
||||
switch (pixel_format) {
|
||||
case kCVPixelFormatType_422YpCbCr8:
|
||||
case kCVPixelFormatType_4444YpCbCrA8:
|
||||
case kCVPixelFormatType_4444YpCbCrA8R:
|
||||
case kCVPixelFormatType_4444AYpCbCr8:
|
||||
case kCVPixelFormatType_4444AYpCbCr16:
|
||||
case kCVPixelFormatType_444YpCbCr8:
|
||||
case kCVPixelFormatType_422YpCbCr16:
|
||||
case kCVPixelFormatType_422YpCbCr10:
|
||||
case kCVPixelFormatType_444YpCbCr10:
|
||||
case kCVPixelFormatType_420YpCbCr8Planar:
|
||||
case kCVPixelFormatType_420YpCbCr8PlanarFullRange:
|
||||
case kCVPixelFormatType_422YpCbCr_4A_8BiPlanar:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_422YpCbCr8_yuvs:
|
||||
case kCVPixelFormatType_422YpCbCr8FullRange:
|
||||
return CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
|
||||
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
vImageConverterRef vImageConverterForCVPixelFormats(OSType src_pixel_format,
|
||||
OSType dst_pixel_format,
|
||||
vImage_Error* error) {
|
||||
static CGFloat default_background[3] = {1.0, 1.0, 1.0};
|
||||
vImageConverterRef converter = NULL;
|
||||
|
||||
vImage_CGImageFormat src_cg_format =
|
||||
vImageFormatForCVPixelFormat(src_pixel_format);
|
||||
vImage_CGImageFormat dst_cg_format =
|
||||
vImageFormatForCVPixelFormat(dst_pixel_format);
|
||||
|
||||
// Use CV format functions if available (introduced in iOS 8).
|
||||
// Weak-linked symbols are NULL when not available.
|
||||
if (&vImageConverter_CreateForCGToCVImageFormat != NULL) {
|
||||
// Strangely, there is no function to convert between two
|
||||
// vImageCVImageFormat, so one side has to use a vImage_CGImageFormat
|
||||
// that we have to find ourselves.
|
||||
if (src_cg_format.bitsPerComponent > 0) {
|
||||
// We can handle source using a CGImageFormat.
|
||||
// TODO: check the final alpha hint parameter
|
||||
CGColorSpaceRef cv_color_space =
|
||||
CreateConversionCGColorSpaceForPixelFormat(dst_pixel_format);
|
||||
vImageCVImageFormatRef dst_cv_format = vImageCVImageFormat_Create(
|
||||
dst_pixel_format, kvImage_ARGBToYpCbCrMatrix_ITU_R_709_2,
|
||||
kCVImageBufferChromaLocation_Center, cv_color_space, 1);
|
||||
CGColorSpaceRelease(cv_color_space);
|
||||
|
||||
converter = vImageConverter_CreateForCGToCVImageFormat(
|
||||
&src_cg_format, dst_cv_format, default_background,
|
||||
kvImagePrintDiagnosticsToConsole, error);
|
||||
vImageCVImageFormat_Release(dst_cv_format);
|
||||
} else if (dst_cg_format.bitsPerComponent > 0) {
|
||||
// We can use a CGImageFormat for the destination.
|
||||
CGColorSpaceRef cv_color_space =
|
||||
CreateConversionCGColorSpaceForPixelFormat(src_pixel_format);
|
||||
vImageCVImageFormatRef src_cv_format = vImageCVImageFormat_Create(
|
||||
src_pixel_format, kvImage_ARGBToYpCbCrMatrix_ITU_R_709_2,
|
||||
kCVImageBufferChromaLocation_Center, cv_color_space, 1);
|
||||
CGColorSpaceRelease(cv_color_space);
|
||||
|
||||
converter = vImageConverter_CreateForCVToCGImageFormat(
|
||||
src_cv_format, &dst_cg_format, default_background,
|
||||
kvImagePrintDiagnosticsToConsole, error);
|
||||
vImageCVImageFormat_Release(src_cv_format);
|
||||
}
|
||||
}
|
||||
|
||||
if (!converter) {
|
||||
// Try a CG to CG conversion.
|
||||
if (src_cg_format.bitsPerComponent > 0 &&
|
||||
dst_cg_format.bitsPerComponent > 0) {
|
||||
converter = vImageConverter_CreateWithCGImageFormat(
|
||||
&src_cg_format, &dst_cg_format, default_background, kvImageNoFlags,
|
||||
error);
|
||||
}
|
||||
}
|
||||
|
||||
CGColorSpaceRelease(src_cg_format.colorSpace);
|
||||
CGColorSpaceRelease(dst_cg_format.colorSpace);
|
||||
return converter;
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
vImage_Error vImageGrayToBGRA(const vImage_Buffer* src, vImage_Buffer* dst) {
|
||||
static vImageConverterRef converter = NULL;
|
||||
if (!converter) {
|
||||
converter = vImageConverterForCVPixelFormats(
|
||||
kCVPixelFormatType_OneComponent8, kCVPixelFormatType_32BGRA, NULL);
|
||||
}
|
||||
return vImageConvert_AnyToAny(converter, src, dst, NULL, kvImageNoFlags);
|
||||
}
|
||||
|
||||
vImage_Error vImageBGRAToGray(const vImage_Buffer* src, vImage_Buffer* dst) {
|
||||
static vImageConverterRef converter = NULL;
|
||||
if (!converter) {
|
||||
converter = vImageConverterForCVPixelFormats(
|
||||
kCVPixelFormatType_32BGRA, kCVPixelFormatType_OneComponent8, NULL);
|
||||
}
|
||||
return vImageConvert_AnyToAny(converter, src, dst, NULL, kvImageNoFlags);
|
||||
}
|
||||
|
||||
vImage_Error vImageRGBAToGray(const vImage_Buffer* src, vImage_Buffer* dst) {
|
||||
static vImageConverterRef converter = NULL;
|
||||
if (!converter) {
|
||||
converter = vImageConverterForCVPixelFormats(
|
||||
kCVPixelFormatType_32RGBA, kCVPixelFormatType_OneComponent8, NULL);
|
||||
}
|
||||
return vImageConvert_AnyToAny(converter, src, dst, NULL, kvImageNoFlags);
|
||||
}
|
||||
|
||||
vImage_Error vImageConvertCVPixelBuffers(CVPixelBufferRef src,
|
||||
CVPixelBufferRef dst) {
|
||||
// CGColorSpaceRef srgb_color_space =
|
||||
// CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
|
||||
vImage_Error error;
|
||||
vImageConverterRef converter = vImageConverterForCVPixelFormats(
|
||||
CVPixelBufferGetPixelFormatType(src),
|
||||
CVPixelBufferGetPixelFormatType(dst), &error);
|
||||
if (!converter) {
|
||||
return error;
|
||||
}
|
||||
|
||||
int src_buffer_count = vImageConverter_GetNumberOfSourceBuffers(converter);
|
||||
int dst_buffer_count =
|
||||
vImageConverter_GetNumberOfDestinationBuffers(converter);
|
||||
vImage_Buffer buffers[8];
|
||||
if (src_buffer_count + dst_buffer_count > ABSL_ARRAYSIZE(buffers)) {
|
||||
vImageConverter_Release(converter);
|
||||
return kvImageMemoryAllocationError;
|
||||
}
|
||||
vImage_Buffer* src_bufs = buffers;
|
||||
vImage_Buffer* dst_bufs = buffers + src_buffer_count;
|
||||
|
||||
// vImageBuffer_InitForCopyToCVPixelBuffer can be used only if the converter
|
||||
// was created by vImageConverter_CreateForCGToCVImageFormat.
|
||||
// vImageBuffer_InitForCopyFromCVPixelBuffer can be used only if the converter
|
||||
// was created by vImageConverter_CreateForCVToCGImageFormat.
|
||||
// There does not seem to be a way to ask the converter for its type; however,
|
||||
// it is documented that all multi-planar formats are CV formats, so we use
|
||||
// these calls when there are multiple buffers.
|
||||
|
||||
if (src_buffer_count > 1) {
|
||||
error = vImageBuffer_InitForCopyFromCVPixelBuffer(
|
||||
src_bufs, converter, src,
|
||||
kvImageNoAllocate | kvImagePrintDiagnosticsToConsole);
|
||||
if (error != kvImageNoError) {
|
||||
vImageConverter_Release(converter);
|
||||
return error;
|
||||
}
|
||||
} else {
|
||||
*src_bufs = vImageForCVPixelBuffer(src);
|
||||
}
|
||||
|
||||
if (dst_buffer_count > 1) {
|
||||
error = vImageBuffer_InitForCopyToCVPixelBuffer(
|
||||
dst_bufs, converter, dst,
|
||||
kvImageNoAllocate | kvImagePrintDiagnosticsToConsole);
|
||||
if (error != kvImageNoError) {
|
||||
vImageConverter_Release(converter);
|
||||
return error;
|
||||
}
|
||||
} else {
|
||||
*dst_bufs = vImageForCVPixelBuffer(dst);
|
||||
}
|
||||
|
||||
error = vImageConvert_AnyToAny(converter, src_bufs, dst_bufs, NULL,
|
||||
kvImageNoFlags);
|
||||
vImageConverter_Release(converter);
|
||||
return error;
|
||||
}
|
||||
|
||||
void ReleaseMediaPipePacket(void* refcon, const void* base_address) {
|
||||
auto packet = (mediapipe::Packet*)refcon;
|
||||
delete packet;
|
||||
}
|
||||
|
||||
CVPixelBufferRef CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet) {
|
||||
CFHolder<CVPixelBufferRef> buffer;
|
||||
::mediapipe::Status status =
|
||||
CreateCVPixelBufferForImageFramePacket(image_frame_packet, &buffer);
|
||||
MEDIAPIPE_CHECK_OK(status) << "Failed to create CVPixelBufferRef";
|
||||
return (CVPixelBufferRef)CFRetain(*buffer);
|
||||
}
|
||||
|
||||
::mediapipe::Status CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet,
|
||||
CFHolder<CVPixelBufferRef>* out_buffer) {
|
||||
return CreateCVPixelBufferForImageFramePacket(image_frame_packet, false,
|
||||
out_buffer);
|
||||
}
|
||||
|
||||
::mediapipe::Status CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet, bool can_overwrite,
|
||||
CFHolder<CVPixelBufferRef>* out_buffer) {
|
||||
if (!out_buffer) {
|
||||
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "out_buffer cannot be NULL";
|
||||
}
|
||||
CFHolder<CVPixelBufferRef> pixel_buffer;
|
||||
|
||||
auto packet_copy = absl::make_unique<mediapipe::Packet>(image_frame_packet);
|
||||
const auto& frame = packet_copy->Get<mediapipe::ImageFrame>();
|
||||
void* frame_data =
|
||||
const_cast<void*>(reinterpret_cast<const void*>(frame.PixelData()));
|
||||
|
||||
mediapipe::ImageFormat::Format image_format = frame.Format();
|
||||
OSType pixel_format = 0;
|
||||
CVReturn status;
|
||||
switch (image_format) {
|
||||
case mediapipe::ImageFormat::SRGBA: {
|
||||
pixel_format = kCVPixelFormatType_32BGRA;
|
||||
// Swap R and B channels.
|
||||
vImage_Buffer v_image = vImageForImageFrame(frame);
|
||||
vImage_Buffer v_dest;
|
||||
if (can_overwrite) {
|
||||
v_dest = v_image;
|
||||
} else {
|
||||
CVPixelBufferRef pixel_buffer_temp;
|
||||
status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault, frame.Width(), frame.Height(), pixel_format,
|
||||
GetCVPixelBufferAttributesForGlCompatibility(), &pixel_buffer_temp);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferCreate failed: " << status;
|
||||
pixel_buffer.adopt(pixel_buffer_temp);
|
||||
status = CVPixelBufferLockBaseAddress(*pixel_buffer,
|
||||
kCVPixelBufferLock_ReadOnly);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferLockBaseAddress failed: " << status;
|
||||
v_dest = vImageForCVPixelBuffer(*pixel_buffer);
|
||||
}
|
||||
const uint8_t permute_map[4] = {2, 1, 0, 3};
|
||||
vImage_Error vError = vImagePermuteChannels_ARGB8888(
|
||||
&v_image, &v_dest, permute_map, kvImageNoFlags);
|
||||
RET_CHECK(vError == kvImageNoError)
|
||||
<< "vImagePermuteChannels failed: " << vError;
|
||||
} break;
|
||||
|
||||
case mediapipe::ImageFormat::GRAY8:
|
||||
pixel_format = kCVPixelFormatType_OneComponent8;
|
||||
break;
|
||||
|
||||
default:
|
||||
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "unsupported ImageFrame format: " << image_format;
|
||||
}
|
||||
|
||||
if (*pixel_buffer) {
|
||||
status = CVPixelBufferUnlockBaseAddress(*pixel_buffer,
|
||||
kCVPixelBufferLock_ReadOnly);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferUnlockBaseAddress failed: " << status;
|
||||
} else {
|
||||
CVPixelBufferRef pixel_buffer_temp;
|
||||
status = CVPixelBufferCreateWithBytes(
|
||||
NULL, frame.Width(), frame.Height(), pixel_format, frame_data,
|
||||
frame.WidthStep(), ReleaseMediaPipePacket, packet_copy.release(),
|
||||
GetCVPixelBufferAttributesForGlCompatibility(), &pixel_buffer_temp);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "failed to create pixel buffer: " << status;
|
||||
pixel_buffer.adopt(pixel_buffer_temp);
|
||||
}
|
||||
|
||||
*out_buffer = pixel_buffer;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status CreateCGImageFromCVPixelBuffer(
|
||||
CVPixelBufferRef image_buffer, CFHolder<CGImageRef>* image) {
|
||||
CVReturn status =
|
||||
CVPixelBufferLockBaseAddress(image_buffer, kCVPixelBufferLock_ReadOnly);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferLockBaseAddress failed: " << status;
|
||||
|
||||
void* base_address = CVPixelBufferGetBaseAddress(image_buffer);
|
||||
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(image_buffer);
|
||||
size_t width = CVPixelBufferGetWidth(image_buffer);
|
||||
size_t height = CVPixelBufferGetHeight(image_buffer);
|
||||
OSType pixel_format = CVPixelBufferGetPixelFormatType(image_buffer);
|
||||
|
||||
CGColorSpaceRef color_space = nullptr;
|
||||
uint32_t bitmap_info = 0;
|
||||
switch (pixel_format) {
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
color_space = CGColorSpaceCreateDeviceRGB();
|
||||
bitmap_info =
|
||||
kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst;
|
||||
break;
|
||||
|
||||
case kCVPixelFormatType_OneComponent8:
|
||||
color_space = CGColorSpaceCreateDeviceGray();
|
||||
bitmap_info = kCGImageAlphaNone;
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG(FATAL) << "Unsupported pixelFormat " << pixel_format;
|
||||
break;
|
||||
}
|
||||
|
||||
CGContextRef src_context = CGBitmapContextCreate(
|
||||
base_address, width, height, 8, bytes_per_row, color_space, bitmap_info);
|
||||
|
||||
CGImageRef quartz_image = CGBitmapContextCreateImage(src_context);
|
||||
CGContextRelease(src_context);
|
||||
CGColorSpaceRelease(color_space);
|
||||
CFHolder<CGImageRef> cg_image_holder = MakeCFHolderAdopting(quartz_image);
|
||||
status =
|
||||
CVPixelBufferUnlockBaseAddress(image_buffer, kCVPixelBufferLock_ReadOnly);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferUnlockBaseAddress failed: " << status;
|
||||
|
||||
*image = cg_image_holder;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status CreateCVPixelBufferFromCGImage(
|
||||
CGImageRef image, CFHolder<CVPixelBufferRef>* out_buffer) {
|
||||
size_t width = CGImageGetWidth(image);
|
||||
size_t height = CGImageGetHeight(image);
|
||||
CFHolder<CVPixelBufferRef> pixel_buffer;
|
||||
|
||||
CVPixelBufferRef pixel_buffer_temp;
|
||||
CVReturn status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA,
|
||||
GetCVPixelBufferAttributesForGlCompatibility(), &pixel_buffer_temp);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "failed to create pixel buffer: " << status;
|
||||
pixel_buffer.adopt(pixel_buffer_temp);
|
||||
|
||||
status = CVPixelBufferLockBaseAddress(*pixel_buffer, 0);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferLockBaseAddress failed: " << status;
|
||||
|
||||
void* base_address = CVPixelBufferGetBaseAddress(*pixel_buffer);
|
||||
CGColorSpaceRef color_space = CGColorSpaceCreateDeviceRGB();
|
||||
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(*pixel_buffer);
|
||||
CGContextRef context = CGBitmapContextCreate(
|
||||
base_address, width, height, 8, bytes_per_row, color_space,
|
||||
kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst);
|
||||
CGRect rect = CGRectMake(0, 0, width, height);
|
||||
CGContextClearRect(context, rect);
|
||||
CGContextDrawImage(context, rect, image);
|
||||
|
||||
CGContextRelease(context);
|
||||
CGColorSpaceRelease(color_space);
|
||||
status = CVPixelBufferUnlockBaseAddress(*pixel_buffer, 0);
|
||||
RET_CHECK(status == kCVReturnSuccess)
|
||||
<< "CVPixelBufferUnlockBaseAddress failed: " << status;
|
||||
|
||||
*out_buffer = pixel_buffer;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
std::unique_ptr<mediapipe::ImageFrame> CreateImageFrameForCVPixelBuffer(
|
||||
CVPixelBufferRef image_buffer) {
|
||||
return CreateImageFrameForCVPixelBuffer(image_buffer, false, false);
|
||||
}
|
||||
|
||||
std::unique_ptr<mediapipe::ImageFrame> CreateImageFrameForCVPixelBuffer(
|
||||
CVPixelBufferRef image_buffer, bool can_overwrite, bool bgr_as_rgb) {
|
||||
CVReturn status =
|
||||
CVPixelBufferLockBaseAddress(image_buffer, kCVPixelBufferLock_ReadOnly);
|
||||
CHECK_EQ(status, kCVReturnSuccess)
|
||||
<< "CVPixelBufferLockBaseAddress failed: " << status;
|
||||
|
||||
void* base_address = CVPixelBufferGetBaseAddress(image_buffer);
|
||||
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(image_buffer);
|
||||
size_t width = CVPixelBufferGetWidth(image_buffer);
|
||||
size_t height = CVPixelBufferGetHeight(image_buffer);
|
||||
std::unique_ptr<mediapipe::ImageFrame> frame;
|
||||
|
||||
CVPixelBufferRetain(image_buffer);
|
||||
|
||||
OSType pixel_format = CVPixelBufferGetPixelFormatType(image_buffer);
|
||||
mediapipe::ImageFormat::Format image_format = mediapipe::ImageFormat::UNKNOWN;
|
||||
switch (pixel_format) {
|
||||
case kCVPixelFormatType_32BGRA: {
|
||||
image_format = mediapipe::ImageFormat::SRGBA;
|
||||
if (!bgr_as_rgb) {
|
||||
// Swap R and B channels.
|
||||
vImage_Buffer v_image = vImageForCVPixelBuffer(image_buffer);
|
||||
vImage_Buffer v_dest;
|
||||
if (can_overwrite) {
|
||||
v_dest = v_image;
|
||||
} else {
|
||||
frame = absl::make_unique<mediapipe::ImageFrame>(image_format, width,
|
||||
height);
|
||||
v_dest = vImageForImageFrame(*frame);
|
||||
}
|
||||
const uint8_t permute_map[4] = {2, 1, 0, 3};
|
||||
vImage_Error vError = vImagePermuteChannels_ARGB8888(
|
||||
&v_image, &v_dest, permute_map, kvImageNoFlags);
|
||||
CHECK(vError == kvImageNoError)
|
||||
<< "vImagePermuteChannels failed: " << vError;
|
||||
}
|
||||
} break;
|
||||
|
||||
case kCVPixelFormatType_32RGBA:
|
||||
image_format = mediapipe::ImageFormat::SRGBA;
|
||||
break;
|
||||
|
||||
case kCVPixelFormatType_24RGB:
|
||||
image_format = mediapipe::ImageFormat::SRGB;
|
||||
break;
|
||||
|
||||
case kCVPixelFormatType_OneComponent8:
|
||||
image_format = mediapipe::ImageFormat::GRAY8;
|
||||
break;
|
||||
|
||||
default: {
|
||||
char format_str[5] = {static_cast<char>(pixel_format >> 24 & 0xFF),
|
||||
static_cast<char>(pixel_format >> 16 & 0xFF),
|
||||
static_cast<char>(pixel_format >> 8 & 0xFF),
|
||||
static_cast<char>(pixel_format & 0xFF), 0};
|
||||
LOG(FATAL) << "unsupported pixel format: " << format_str;
|
||||
} break;
|
||||
}
|
||||
|
||||
if (frame) {
|
||||
// We have already created a new frame that does not reference the buffer.
|
||||
status = CVPixelBufferUnlockBaseAddress(image_buffer,
|
||||
kCVPixelBufferLock_ReadOnly);
|
||||
CHECK_EQ(status, kCVReturnSuccess)
|
||||
<< "CVPixelBufferUnlockBaseAddress failed: " << status;
|
||||
CVPixelBufferRelease(image_buffer);
|
||||
} else {
|
||||
frame = absl::make_unique<mediapipe::ImageFrame>(
|
||||
image_format, width, height, bytes_per_row,
|
||||
reinterpret_cast<uint8*>(base_address), [image_buffer](uint8* x) {
|
||||
CVPixelBufferUnlockBaseAddress(image_buffer,
|
||||
kCVPixelBufferLock_ReadOnly);
|
||||
CVPixelBufferRelease(image_buffer);
|
||||
});
|
||||
}
|
||||
return frame;
|
||||
}
|
||||
|
||||
CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
|
||||
static CFDictionaryRef attrs = NULL;
|
||||
if (!attrs) {
|
||||
CFDictionaryRef empty_dict = CFDictionaryCreate(
|
||||
kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
// To ensure compatibility with CVOpenGLESTextureCache, these attributes
|
||||
// should be present.
|
||||
const void* keys[] = {
|
||||
kCVPixelBufferIOSurfacePropertiesKey,
|
||||
#if TARGET_OS_OSX
|
||||
kCVPixelFormatOpenGLCompatibility,
|
||||
#else
|
||||
kCVPixelFormatOpenGLESCompatibility,
|
||||
#endif // TARGET_OS_OSX
|
||||
};
|
||||
const void* values[] = {empty_dict, kCFBooleanTrue};
|
||||
attrs = CFDictionaryCreate(
|
||||
kCFAllocatorDefault, keys, values, ABSL_ARRAYSIZE(values),
|
||||
&kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
|
||||
CFRelease(empty_dict);
|
||||
}
|
||||
return attrs;
|
||||
}
|
||||
|
||||
void DumpCVPixelFormats() {
|
||||
CFArrayRef pf_descs =
|
||||
CVPixelFormatDescriptionArrayCreateWithAllPixelFormatTypes(
|
||||
kCFAllocatorDefault);
|
||||
CFIndex count = CFArrayGetCount(pf_descs);
|
||||
CFIndex i;
|
||||
|
||||
printf("Core Video Supported Pixel Format Types:\n");
|
||||
|
||||
for (i = 0; i < count; i++) {
|
||||
CFNumberRef pf_num = (CFNumberRef)CFArrayGetValueAtIndex(pf_descs, i);
|
||||
if (!pf_num) continue;
|
||||
|
||||
int pf;
|
||||
CFNumberGetValue(pf_num, kCFNumberSInt32Type, &pf);
|
||||
|
||||
if (pf <= 0x28) {
|
||||
printf("\nCore Video Pixel Format Type: %d\n", pf);
|
||||
} else {
|
||||
printf("\nCore Video Pixel Format Type (FourCC): %c%c%c%c\n",
|
||||
static_cast<char>(pf >> 24), static_cast<char>(pf >> 16),
|
||||
static_cast<char>(pf >> 8), static_cast<char>(pf));
|
||||
}
|
||||
|
||||
CFDictionaryRef desc = CVPixelFormatDescriptionCreateWithPixelFormatType(
|
||||
kCFAllocatorDefault, pf);
|
||||
CFDictionaryApplyFunction(
|
||||
desc,
|
||||
[](const void* key, const void* value, void* context) {
|
||||
CFStringRef s = CFStringCreateWithFormat(
|
||||
kCFAllocatorDefault, nullptr, CFSTR(" %@: %@"), key, value);
|
||||
CFShow(s);
|
||||
CFRelease(s);
|
||||
},
|
||||
nullptr);
|
||||
CFRelease(desc);
|
||||
}
|
||||
CFRelease(pf_descs);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_OBJC_UTIL_H_
|
||||
#define MEDIAPIPE_OBJC_UTIL_H_
|
||||
|
||||
#import <Accelerate/Accelerate.h>
|
||||
#import <CoreFoundation/CoreFoundation.h>
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/objc/CFHolder.h"
|
||||
|
||||
// TODO: namespace and/or prefix these. Split up the file.
|
||||
|
||||
/// Returns a vImage_Buffer describing the data inside the pixel_buffer.
|
||||
/// NOTE: the pixel buffer's base address must have been locked before this
|
||||
/// call, and it must stay locked as long as the vImage_Buffer is in use.
|
||||
inline vImage_Buffer vImageForCVPixelBuffer(CVPixelBufferRef pixel_buffer) {
|
||||
return {.data = CVPixelBufferGetBaseAddress(pixel_buffer),
|
||||
.width = CVPixelBufferGetWidth(pixel_buffer),
|
||||
.height = CVPixelBufferGetHeight(pixel_buffer),
|
||||
.rowBytes = CVPixelBufferGetBytesPerRow(pixel_buffer)};
|
||||
}
|
||||
|
||||
/// Returns a vImage_Buffer describing the data inside the ImageFrame.
|
||||
inline vImage_Buffer vImageForImageFrame(const mediapipe::ImageFrame& frame) {
|
||||
return {.data = (void*)frame.PixelData(),
|
||||
.width = static_cast<vImagePixelCount>(frame.Width()),
|
||||
.height = static_cast<vImagePixelCount>(frame.Height()),
|
||||
.rowBytes = static_cast<size_t>(frame.WidthStep())};
|
||||
}
|
||||
|
||||
/// Converts a grayscale image without alpha to BGRA format.
|
||||
vImage_Error vImageGrayToBGRA(const vImage_Buffer* src, vImage_Buffer* dst);
|
||||
|
||||
/// Converts a BGRA image to grayscale without alpha.
|
||||
vImage_Error vImageBGRAToGray(const vImage_Buffer* src, vImage_Buffer* dst);
|
||||
|
||||
/// Converts an RGBA image to grayscale without alpha.
|
||||
vImage_Error vImageRGBAToGray(const vImage_Buffer* src, vImage_Buffer* dst);
|
||||
|
||||
/// Copy from a pixel buffer to another, converting pixel format.
|
||||
/// Both pixel buffers should be locked before calling this.
|
||||
vImage_Error vImageConvertCVPixelBuffers(CVPixelBufferRef src,
|
||||
CVPixelBufferRef dst);
|
||||
|
||||
/// When storing a mediapipe::Packet* in a CVPixelBuffer's refcon, this can be
|
||||
/// used as a CVPixelBufferReleaseBytesCallback. This keeps the packet's data
|
||||
/// alive while the CVPixelBuffer is in use.
|
||||
void ReleaseMediaPipePacket(void* refcon, const void* base_address);
|
||||
|
||||
/// Returns a CVPixelBuffer that references the data inside the packet. The
|
||||
/// packet must contain an ImageFrame. The CVPixelBuffer manages a copy of
|
||||
/// the packet, so that the packet's data is kept alive as long as the
|
||||
/// CVPixelBuffer is in use.
|
||||
///
|
||||
/// For formats which are not supported by both image types, it may be
|
||||
/// necessary to convert the data. This is done by creating a new buffer.
|
||||
/// If the optional can_overwrite parameter is true, the old buffer may be
|
||||
/// modified instead.
|
||||
::mediapipe::Status CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet,
|
||||
CFHolder<CVPixelBufferRef>* out_buffer);
|
||||
::mediapipe::Status CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet, bool can_overwrite,
|
||||
CFHolder<CVPixelBufferRef>* out_buffer);
|
||||
|
||||
/// Creates a CVPixelBuffer with a copy of the contents of the CGImage.
|
||||
::mediapipe::Status CreateCVPixelBufferFromCGImage(
|
||||
CGImageRef image, CFHolder<CVPixelBufferRef>* out_buffer);
|
||||
|
||||
/// Creates a CGImage with a copy of the contents of the CVPixelBuffer.
|
||||
::mediapipe::Status CreateCGImageFromCVPixelBuffer(
|
||||
CVPixelBufferRef image_buffer, CFHolder<CGImageRef>* image);
|
||||
|
||||
/// DEPRECATED: use the version that returns ::mediapipe::Status instead.
|
||||
CVPixelBufferRef CreateCVPixelBufferForImageFramePacket(
|
||||
const mediapipe::Packet& image_frame_packet);
|
||||
|
||||
/// Returns an ImageFrame that references the data inside the pixel_buffer.
|
||||
/// The ImageFrame retains the pixel_buffer and keeps it locked as long as it
|
||||
/// is in use.
|
||||
///
|
||||
/// For formats which are not supported by both image types, it may be
|
||||
/// necessary to convert the data. This is done by creating a new buffer.
|
||||
/// If the optional can_overwrite parameter is true, the old buffer may be
|
||||
/// modified instead.
|
||||
///
|
||||
/// ImageFrame does not have a format for BGRA data, so we normally swap the
|
||||
/// channels to produce RGBA. But many graphs do not care about the order of
|
||||
/// the channels; in those cases, setting the optional bgr_as_rgb parameter
|
||||
/// to true skips the channel swap.
|
||||
std::unique_ptr<mediapipe::ImageFrame> CreateImageFrameForCVPixelBuffer(
|
||||
CVPixelBufferRef pixel_buffer);
|
||||
std::unique_ptr<mediapipe::ImageFrame> CreateImageFrameForCVPixelBuffer(
|
||||
CVPixelBufferRef pixel_buffer, bool can_overwrite, bool bgr_as_rgb);
|
||||
|
||||
/// Returns a CFDictionaryRef that can be passed to CVPixelBufferCreate to
|
||||
/// ensure that the pixel buffer is compatible with OpenGL ES and with
|
||||
/// CVOpenGLESTextureCacheCreateTextureFromImage.
|
||||
/// The returned object is persistent and should not be released.
|
||||
CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility();
|
||||
|
||||
/// Prints debug information about available CoreVideo pixel formats.
|
||||
/// This prints to stdout.
|
||||
void DumpCVPixelFormats();
|
||||
|
||||
#endif // MEDIAPIPE_OBJC_UTIL_H_
|
||||
Reference in New Issue
Block a user