talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 73d686c40057684f8bfaca285368bf1813f9fc26
This commit is contained in:
MediaPipe Team
2022-03-21 12:12:39 -07:00
committed by jqtang
parent e6c19885c6
commit cc6a2f7af6
266 changed files with 3658 additions and 1681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/image/BUILD
+3
View File
@@ -239,10 +239,13 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":image_transformation_calculator_cc_proto",
"//mediapipe/framework:packet",
"//mediapipe/framework:timestamp",
"//mediapipe/gpu:scale_mode_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
mediapipe/calculators/image/image_file_properties_calculator.cc
+1 -1
View File
@@ -105,7 +105,7 @@ absl::StatusOr<ImageFileProperties> GetImageFileProperites(
} // namespace
// Calculator to extract EXIF information from an image file. The input is
// a std::string containing raw byte data from a file, and the output is an
// a string containing raw byte data from a file, and the output is an
// ImageFileProperties proto object with the relevant fields filled in.
// The calculator accepts the input as a stream or a side packet, and can output
// the result as a stream or a side packet. The calculator checks that if an
mediapipe/calculators/image/image_transformation_calculator.cc
+43
View File
@@ -16,10 +16,13 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/gpu/scale_mode.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
@@ -52,6 +55,7 @@ namespace mediapipe {
namespace {
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
switch (rotation) {
@@ -122,6 +126,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// provided, it overrides the FLIP_VERTICALLY input side packet and/or
// corresponding field in the calculator options.
//
// VIDEO_PRESTREAM (optional): VideoHeader for the input ImageFrames, if
// rotating or scaling the frames, the header width and height will be updated
// appropriately. Note the header is updated only based on dimensions and
// rotations specified as side packets or options, input_stream
// transformations will not update the header.
//
// Output:
// One of the following tags:
// IMAGE - ImageFrame representing the output image.
@@ -242,6 +252,21 @@ absl::Status ImageTransformationCalculator::GetContract(
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
RET_CHECK(cc->Inputs().HasTag(kVideoPrestreamTag) ==
cc->Outputs().HasTag(kVideoPrestreamTag))
<< "If VIDEO_PRESTREAM is provided, it must be provided both as an "
"inputs and output stream.";
if (cc->Inputs().HasTag(kVideoPrestreamTag)) {
RET_CHECK(!(cc->Inputs().HasTag("OUTPUT_DIMENSIONS") ||
cc->Inputs().HasTag("ROTATION_DEGREES")))
<< "If specifying VIDEO_PRESTREAM, the transformations that affect the "
"dimensions of the frames (OUTPUT_DIMENSIONS and ROTATION_DEGREES) "
"need to be constant for every frame, meaning they can only be "
"provided in the calculator options or side packets.";
cc->Inputs().Tag(kVideoPrestreamTag).Set<mediapipe::VideoHeader>();
cc->Outputs().Tag(kVideoPrestreamTag).Set<mediapipe::VideoHeader>();
}
if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
}
@@ -326,6 +351,24 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
}
absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
// First update the video header if it is given, based on the rotation and
// dimensions specified as side packets or options. This will only be done
// once, so streaming transformation changes will not be reflected in
// the header.
if (cc->Inputs().HasTag(kVideoPrestreamTag) &&
!cc->Inputs().Tag(kVideoPrestreamTag).IsEmpty() &&
cc->Outputs().HasTag(kVideoPrestreamTag)) {
mediapipe::VideoHeader header =
cc->Inputs().Tag(kVideoPrestreamTag).Get<mediapipe::VideoHeader>();
// Update the header's width and height if needed.
ComputeOutputDimensions(header.width, header.height, &header.width,
&header.height);
cc->Outputs()
.Tag(kVideoPrestreamTag)
.AddPacket(mediapipe::MakePacket<mediapipe::VideoHeader>(header).At(
mediapipe::Timestamp::PreStream()));
}
// Override values if specified so.
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
!cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc
+3 -3
View File
@@ -22,9 +22,9 @@
namespace mediapipe {
// Takes in an encoded image std::string, decodes it by OpenCV, and converts to
// an ImageFrame. Note that this calculator only supports grayscale and RGB
// images for now.
// Takes in an encoded image string, decodes it by OpenCV, and converts to an
// ImageFrame. Note that this calculator only supports grayscale and RGB images
// for now.
//
// Example config:
// node {
mediapipe/calculators/image/opencv_put_text_calculator.cc
+2 -2
View File
@@ -20,8 +20,8 @@
namespace mediapipe {
// Takes in a std::string, draws the text std::string by cv::putText(), and
// outputs an ImageFrame.
// Takes in a string, draws the text string by cv::putText(), and outputs an
// ImageFrame.
//
// Example config:
// node {
mediapipe/calculators/image/scale_image_calculator.cc
-1
View File
@@ -553,7 +553,6 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
}
}
cc->GetCounter("Inputs")->Increment();
const ImageFrame* image_frame;
ImageFrame converted_image_frame;
if (input_format_ == ImageFormat::YCBCR420P) {
mediapipe/calculators/image/segmentation_smoothing_calculator.cc
+11 -11
View File
@@ -183,22 +183,22 @@ absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) {
absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
// Setup source images.
const auto& current_frame = cc->Inputs().Tag(kCurrentMaskTag).Get<Image>();
const cv::Mat current_mat = mediapipe::formats::MatView(&current_frame);
RET_CHECK_EQ(current_mat.type(), CV_32FC1)
auto current_mat = mediapipe::formats::MatView(&current_frame);
RET_CHECK_EQ(current_mat->type(), CV_32FC1)
<< "Only 1-channel float input image is supported.";
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
const cv::Mat previous_mat = mediapipe::formats::MatView(&previous_frame);
RET_CHECK_EQ(previous_mat.type(), current_mat.type())
<< "Warning: mixing input format types: " << previous_mat.type()
<< " != " << previous_mat.type();
auto previous_mat = mediapipe::formats::MatView(&previous_frame);
RET_CHECK_EQ(previous_mat->type(), current_mat->type())
<< "Warning: mixing input format types: " << previous_mat->type()
<< " != " << previous_mat->type();
RET_CHECK_EQ(current_mat.rows, previous_mat.rows);
RET_CHECK_EQ(current_mat.cols, previous_mat.cols);
RET_CHECK_EQ(current_mat->rows, previous_mat->rows);
RET_CHECK_EQ(current_mat->cols, previous_mat->cols);
// Setup destination image.
auto output_frame = std::make_shared<ImageFrame>(
current_frame.image_format(), current_mat.cols, current_mat.rows);
current_frame.image_format(), current_mat->cols, current_mat->rows);
cv::Mat output_mat = mediapipe::formats::MatView(output_frame.get());
output_mat.setTo(cv::Scalar(0));
@@ -233,8 +233,8 @@ absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
// Write directly to the first channel of output.
for (int i = 0; i < output_mat.rows; ++i) {
float* out_ptr = output_mat.ptr<float>(i);
const float* curr_ptr = current_mat.ptr<float>(i);
const float* prev_ptr = previous_mat.ptr<float>(i);
const float* curr_ptr = current_mat->ptr<float>(i);
const float* prev_ptr = previous_mat->ptr<float>(i);
for (int j = 0; j < output_mat.cols; ++j) {
const float new_mask_value = curr_ptr[j];
const float prev_mask_value = prev_ptr[j];
mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc
+4 -4
View File
@@ -116,8 +116,8 @@ void RunGraph(Packet curr_packet, Packet prev_packet, bool use_gpu, float ratio,
ASSERT_EQ(1, output_packets.size());
Image result_image = output_packets[0].Get<Image>();
cv::Mat result_mat = formats::MatView(&result_image);
result_mat.copyTo(*result);
auto result_mat = formats::MatView(&result_image);
result_mat->copyTo(*result);
// Fully close graph at end, otherwise calculator+Images are destroyed
// after calling WaitUntilDone().
@@ -135,10 +135,10 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
Packet curr_packet = MakePacket<Image>(std::make_unique<ImageFrame>(
ImageFormat::VEC32F1, curr_mat.size().width, curr_mat.size().height));
curr_mat.copyTo(formats::MatView(&(curr_packet.Get<Image>())));
curr_mat.copyTo(*formats::MatView(&(curr_packet.Get<Image>())));
Packet prev_packet = MakePacket<Image>(std::make_unique<ImageFrame>(
ImageFormat::VEC32F1, prev_mat.size().width, prev_mat.size().height));
prev_mat.copyTo(formats::MatView(&(prev_packet.Get<Image>())));
prev_mat.copyTo(*formats::MatView(&(prev_packet.Get<Image>())));
cv::Mat result;
RunGraph(curr_packet, prev_packet, use_gpu, mix_ratio, &result);