talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6e5aa035cd1f6a9333962df5d3ab97a05bd5744e
This commit is contained in:
MediaPipe Team
2022-06-28 12:11:05 +00:00
committed by Sebastian Schmidt
parent 4a20e9909d
commit c688862570
144 changed files with 5772 additions and 2118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/image/BUILD
+28 -4
View File
@@ -626,11 +626,8 @@ cc_library(
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
] + select({
@@ -641,6 +638,13 @@ cc_library(
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:shader_util",
],
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/port:opencv_core",
],
}),
alwayslink = 1,
)
@@ -727,7 +731,6 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":affine_transformation",
":affine_transformation_runner_opencv",
":warp_affine_calculator_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -745,6 +748,9 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
":affine_transformation_runner_gl",
],
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [":affine_transformation_runner_opencv"],
}),
alwayslink = 1,
)
@@ -799,3 +805,21 @@ cc_test(
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "yuv_to_image_calculator",
srcs = ["yuv_to_image_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:yuv_image",
"//third_party/libyuv",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
],
alwayslink = 1,
)
mediapipe/calculators/image/segmentation_smoothing_calculator.cc
+12 -3
View File
@@ -21,10 +21,7 @@
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
@@ -34,6 +31,12 @@
#include "mediapipe/gpu/shader_util.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#endif // !MEDIAPIPE_DISABLE_OPENCV
namespace mediapipe {
namespace {
@@ -163,7 +166,11 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) {
return absl::InternalError("GPU processing is disabled.");
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
#if !MEDIAPIPE_DISABLE_OPENCV
MP_RETURN_IF_ERROR(RenderCpu(cc));
#else
return absl::InternalError("OpenCV processing is disabled.");
#endif // !MEDIAPIPE_DISABLE_OPENCV
}
return absl::OkStatus();
@@ -181,6 +188,7 @@ absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) {
}
absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_OPENCV
// Setup source images.
const auto& current_frame = cc->Inputs().Tag(kCurrentMaskTag).Get<Image>();
auto current_mat = mediapipe::formats::MatView(&current_frame);
@@ -245,6 +253,7 @@ absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
cc->Outputs()
.Tag(kOutputMaskTag)
.AddPacket(MakePacket<Image>(output_frame).At(cc->InputTimestamp()));
#endif // !MEDIAPIPE_DISABLE_OPENCV
return absl::OkStatus();
}
mediapipe/calculators/image/warp_affine_calculator.cc
+15 -1
View File
@@ -24,7 +24,9 @@
#endif // !MEDIAPIPE_DISABLE_GPU
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
#endif // !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/calculators/image/warp_affine_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -54,6 +56,7 @@ AffineTransformation::BorderMode GetBorderMode(
template <typename ImageT>
class WarpAffineRunnerHolder {};
#if !MEDIAPIPE_DISABLE_OPENCV
template <>
class WarpAffineRunnerHolder<ImageFrame> {
public:
@@ -69,6 +72,7 @@ class WarpAffineRunnerHolder<ImageFrame> {
private:
std::unique_ptr<RunnerType> runner_;
};
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if !MEDIAPIPE_DISABLE_GPU
template <>
@@ -113,7 +117,9 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
mediapipe::Image> {
public:
absl::Status Open(CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_OPENCV
MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -133,20 +139,26 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
return absl::UnavailableError("GPU support is disabled");
#endif // !MEDIAPIPE_DISABLE_GPU
}
#if !MEDIAPIPE_DISABLE_OPENCV
ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
const auto& frame_ptr = input.GetImageFrameSharedPtr();
// Wrap image into image frame.
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
frame_ptr->Height(), frame_ptr->WidthStep(),
const_cast<uint8_t*>(frame_ptr->PixelData()),
[](uint8* data) {});
[](uint8* data){});
ASSIGN_OR_RETURN(auto result,
runner->Run(image_frame, matrix, size, border_mode));
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
#else
return absl::UnavailableError("OpenCV support is disabled");
#endif // !MEDIAPIPE_DISABLE_OPENCV
}
private:
#if !MEDIAPIPE_DISABLE_OPENCV
WarpAffineRunnerHolder<ImageFrame> cpu_holder_;
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if !MEDIAPIPE_DISABLE_GPU
WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -200,8 +212,10 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
} // namespace
#if !MEDIAPIPE_DISABLE_OPENCV
MEDIAPIPE_NODE_IMPLEMENTATION(
WarpAffineCalculatorImpl<WarpAffineCalculatorCpu>);
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if !MEDIAPIPE_DISABLE_GPU
MEDIAPIPE_NODE_IMPLEMENTATION(
WarpAffineCalculatorImpl<WarpAffineCalculatorGpu>);
mediapipe/calculators/image/warp_affine_calculator.h
+2
View File
@@ -70,11 +70,13 @@ class WarpAffineCalculatorIntf : public mediapipe::api2::NodeIntf {
static constexpr mediapipe::api2::Output<ImageT> kOutImage{"IMAGE"};
};
#if !MEDIAPIPE_DISABLE_OPENCV
class WarpAffineCalculatorCpu : public WarpAffineCalculatorIntf<ImageFrame> {
public:
MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorCpu, kInImage, kMatrix,
kOutputSize, kOutImage);
};
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if !MEDIAPIPE_DISABLE_GPU
class WarpAffineCalculatorGpu
: public WarpAffineCalculatorIntf<mediapipe::GpuBuffer> {
mediapipe/calculators/image/yuv_to_image_calculator.cc
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <utility>
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "libyuv/convert_argb.h"
#include "libyuv/video_common.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/yuv_image.h"
namespace mediapipe {
namespace api2 {
namespace {
// Utility function to convert FourCC enum to string, for error messages.
std::string FourCCToString(libyuv::FourCC fourcc) {
char buf[5];
buf[0] = (fourcc >> 24) & 0xff;
buf[1] = (fourcc >> 16) & 0xff;
buf[2] = (fourcc >> 8) & 0xff;
buf[3] = (fourcc)&0xff;
buf[4] = 0;
return std::string(buf);
}
} // namespace
// Converts a `YUVImage` into an RGB `Image` using libyuv.
//
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
// YV21) format (as per the `fourcc()` property). This covers the most commonly
// used YUV image formats used on mobile devices. Other formats are not
// supported and wil result in an `InvalidArgumentError`.
class YUVToImageCalculator : public Node {
public:
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
static constexpr Output<Image> kOutput{"IMAGE"};
MEDIAPIPE_NODE_CONTRACT(kInput, kOutput);
absl::Status Process(CalculatorContext* cc) override {
const auto& yuv_image = *kInput(cc);
// Check that the format is supported.
auto format = yuv_image.fourcc();
if (format != libyuv::FOURCC_NV12 && format != libyuv::FOURCC_NV21 &&
format != libyuv::FOURCC_YV12 && format != libyuv::FOURCC_I420) {
return absl::InvalidArgumentError(
absl::StrFormat("Unsupported YUVImage format: %s. Only NV12, NV21, "
"YV12 and I420 (aka YV21) are supported.",
FourCCToString(format)));
}
// Build a transient ImageFrameSharedPtr with default alignment to host
// conversion results.
ImageFrameSharedPtr image_frame = std::make_shared<ImageFrame>(
ImageFormat::SRGB, yuv_image.width(), yuv_image.height());
// Perform actual conversion.
switch (format) {
case libyuv::FOURCC_NV12:
// 8-bit Y plane followed by an interleaved 8-bit U/V plane with 2×2
// subsampling.
libyuv::NV12ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), image_frame->MutablePixelData(),
image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_NV21:
// 8-bit Y plane followed by an interleaved 8-bit V/U plane with 2×2
// subsampling.
libyuv::NV21ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), image_frame->MutablePixelData(),
image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_I420:
// Also known as YV21.
// 8-bit Y plane followed by 8-bit 2×2 subsampled U and V planes.
libyuv::I420ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), yuv_image.data(2), yuv_image.stride(2),
image_frame->MutablePixelData(), image_frame->WidthStep(),
yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_YV12:
// 8-bit Y plane followed by 8-bit 2×2 subsampled V and U planes.
libyuv::I420ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(2),
yuv_image.stride(2), yuv_image.data(1), yuv_image.stride(1),
image_frame->MutablePixelData(), image_frame->WidthStep(),
yuv_image.width(), yuv_image.height());
break;
default:
// This should never happen (caught by checks above).
return absl::InternalError("Unsupported YUVImage format.");
}
// Finally, build and send an Image object that takes ownership of the
// transient ImageFrameSharedPtr object.
kOutput(cc).Send(std::make_unique<Image>(std::move(image_frame)));
return absl::OkStatus();
}
};
MEDIAPIPE_REGISTER_NODE(YUVToImageCalculator);
} // namespace api2
} // namespace mediapipe