talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: bb059a0721c92e8154d33ce8057b3915a25b3d7d
This commit is contained in:
MediaPipe Team
2021-12-13 15:56:02 -08:00
committed by jqtang
parent cf101e62a9
commit e6c19885c6
96 changed files with 554 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
-1
View File
@@ -1242,7 +1242,6 @@ cc_test(
"//mediapipe/framework:calculator_profile_cc_proto",
"//mediapipe/framework:test_calculators",
"//mediapipe/framework/deps:clock",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
mediapipe/calculators/core/graph_profile_calculator_test.cc
-1
View File
@@ -23,7 +23,6 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_profile.pb.h"
#include "mediapipe/framework/deps/clock.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
mediapipe/calculators/image/image_transformation_calculator.cc
+3
View File
@@ -45,6 +45,9 @@ namespace mediapipe {
#if !MEDIAPIPE_DISABLE_GPU
#endif // !MEDIAPIPE_DISABLE_GPU
#if defined(MEDIAPIPE_IOS)
#endif // defined(MEDIAPIPE_IOS)
namespace {
constexpr char kImageFrameTag[] = "IMAGE";
mediapipe/calculators/tensor/inference_calculator_test.cc
-2
View File
@@ -39,8 +39,6 @@
namespace mediapipe {
using ::tflite::Interpreter;
void DoSmokeTest(const std::string& graph_proto) {
const int width = 8;
const int height = 8;
mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc
-1
View File
@@ -26,7 +26,6 @@
#include "tensorflow/core/framework/types.h"
namespace mediapipe {
namespace {
constexpr char kTransposeOptionsString[] =
mediapipe/calculators/util/BUILD
+1 -8
View File
@@ -176,7 +176,6 @@ cc_test(
":filter_detections_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
@@ -215,7 +214,6 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/deps:clock",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/stream_handler:immediate_input_stream_handler",
@@ -488,7 +486,6 @@ cc_test(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
@@ -772,7 +769,6 @@ cc_test(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/port:gtest_main",
@@ -869,7 +865,6 @@ cc_test(
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:gtest_main",
@@ -1067,7 +1062,6 @@ cc_test(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
@@ -1217,7 +1211,6 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
@@ -1285,6 +1278,7 @@ cc_library(
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/port:logging",
@@ -1344,7 +1338,6 @@ cc_test(
":detection_classifications_merger_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/port:gtest_main",
mediapipe/calculators/util/association_calculator_test.cc
-1
View File
@@ -16,7 +16,6 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc
+1 -1
View File
@@ -14,9 +14,9 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
mediapipe/calculators/util/detections_to_rects_calculator_test.cc
-1
View File
@@ -19,7 +19,6 @@
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
mediapipe/calculators/util/detections_to_render_data_calculator_test.cc
-3
View File
@@ -17,7 +17,6 @@
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/packet.h"
@@ -34,8 +33,6 @@ constexpr char kDetectionsTag[] = "DETECTIONS";
constexpr char kRenderDataTag[] = "RENDER_DATA";
constexpr char kDetectionListTag[] = "DETECTION_LIST";
using ::testing::DoubleNear;
// Error tolerance for pixels, distances, etc.
static constexpr double kErrorTolerance = 1e-5;
mediapipe/calculators/util/filter_detections_calculator_test.cc
-1
View File
@@ -16,7 +16,6 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/calculators/util/inverse_matrix_calculator.cc
+11 -3
View File
@@ -33,9 +33,17 @@ class InverseMatrixCalculatorImpl : public NodeImpl<InverseMatrixCalculator> {
kInputMatrix(cc).Get().data());
Eigen::Matrix<float, 4, 4, Eigen::RowMajor> inverse_matrix;
bool inverse_check;
matrix.computeInverseWithCheck(inverse_matrix, inverse_check);
RET_CHECK(inverse_check) << "Inverse matrix cannot be calculated.";
bool inverse_check = false;
// The matrix is invertible if the absolute value of its determinant is
// greater than this threshold. Quite small threshold is selected to enable
// inverting valid matrices containing relatively small values resulting in
// a small determinant.
constexpr double kAbsDeterminantThreshold =
Eigen::NumTraits<double>::epsilon();
matrix.computeInverseWithCheck(inverse_matrix, inverse_check,
kAbsDeterminantThreshold);
RET_CHECK(inverse_check)
<< "Inverse matrix cannot be calculated for: " << matrix;
std::array<float, 16> output;
Eigen::Map<Eigen::Matrix<float, 4, 4, Eigen::RowMajor>>(
mediapipe/calculators/util/inverse_matrix_calculator_test.cc
+25 -1
View File
@@ -42,7 +42,11 @@ void RunTest(const std::array<float, 16>& matrix,
const auto& inverse_matrix = output_packets[0].Get<std::array<float, 16>>();
EXPECT_THAT(inverse_matrix, testing::Eq(expected_inverse_matrix));
EXPECT_THAT(
inverse_matrix,
testing::Pointwise(testing::FloatEq(),
absl::MakeSpan(expected_inverse_matrix.data(),
expected_inverse_matrix.size())));
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
@@ -122,5 +126,25 @@ TEST(InverseMatrixCalculatorTest, Rotation90) {
RunTest(matrix, expected_inverse_matrix);
}
TEST(InverseMatrixCalculatorTest, CheckPrecision) {
// clang-format off
std::array<float, 16> matrix = {
0.00001f, 0.0f, 0.0f, 0.0f,
0.0f, 0.00001f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f,
};
std::array<float, 16> expected_inverse_matrix = {
100000.0f, 0.0f, 0.0f, 0.0f,
0.0f, 100000.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f,
};
// clang-format on
RunTest(matrix, expected_inverse_matrix);
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/util/landmark_projection_calculator_test.cc
+1 -1
View File
@@ -6,9 +6,9 @@
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
mediapipe/calculators/util/packet_latency_calculator_test.cc
-1
View File
@@ -16,7 +16,6 @@
#include "mediapipe/calculators/util/latency.pb.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/clock.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
mediapipe/calculators/util/to_image_calculator.cc
+68 -91
View File
@@ -14,6 +14,7 @@
#include <memory>
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image.h"
@@ -23,23 +24,23 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_calculator_helper.h"
#endif // !MEDIAPIPE_DISABLE_GPU
namespace mediapipe {
namespace api2 {
namespace {
constexpr char kImageFrameTag[] = "IMAGE_CPU";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kImageTag[] = "IMAGE";
} // namespace
#if MEDIAPIPE_DISABLE_GPU
// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
class Nothing {};
using GpuBuffer = Nothing;
#else
using GpuBuffer = mediapipe::GpuBuffer;
#endif // MEDIAPIPE_DISABLE_GPU
// A calculator for converting from legacy MediaPipe datatypes into a
// unified image container.
//
// Inputs:
// One of the following two tags:
// IMAGE: An Image, ImageFrame, or GpuBuffer containing input image.
// IMAGE_CPU: An ImageFrame containing input image.
// IMAGE_GPU: A GpuBuffer containing input image.
//
@@ -49,107 +50,44 @@ constexpr char kImageTag[] = "IMAGE";
// Note:
// No CPU/GPU conversion is done.
//
class ToImageCalculator : public CalculatorBase {
class ToImageCalculator : public Node {
public:
ToImageCalculator() = default;
~ToImageCalculator() override = default;
static absl::Status GetContract(CalculatorContract* cc);
static constexpr Input<
OneOf<mediapipe::Image, mediapipe::ImageFrame, GpuBuffer>>::Optional kIn{
"IMAGE"};
static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
static constexpr Output<mediapipe::Image> kOut{"IMAGE"};
MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
static absl::Status UpdateContract(CalculatorContract* cc);
// From Calculator.
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
private:
absl::Status RenderGpu(CalculatorContext* cc);
absl::Status RenderCpu(CalculatorContext* cc);
bool gpu_input_ = false;
bool gpu_initialized_ = false;
#if !MEDIAPIPE_DISABLE_GPU
mediapipe::GlCalculatorHelper gpu_helper_;
#endif // !MEDIAPIPE_DISABLE_GPU
absl::StatusOr<Packet<Image>> GetInputImage(CalculatorContext* cc);
};
REGISTER_CALCULATOR(ToImageCalculator);
MEDIAPIPE_REGISTER_NODE(ToImageCalculator);
absl::Status ToImageCalculator::GetContract(CalculatorContract* cc) {
cc->Outputs().Tag(kImageTag).Set<mediapipe::Image>();
bool gpu_input = false;
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().HasTag(kGpuBufferTag)) {
absl::Status ToImageCalculator::UpdateContract(CalculatorContract* cc) {
int num_inputs = static_cast<int>(kIn(cc).IsConnected()) +
static_cast<int>(kInCpu(cc).IsConnected()) +
static_cast<int>(kInGpu(cc).IsConnected());
if (num_inputs != 1) {
return absl::InternalError("Cannot have multiple inputs.");
}
if (cc->Inputs().HasTag(kGpuBufferTag)) {
#if !MEDIAPIPE_DISABLE_GPU
cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
gpu_input = true;
#else
RET_CHECK_FAIL() << "GPU is disabled. Cannot use IMAGE_GPU stream.";
#endif // !MEDIAPIPE_DISABLE_GPU
}
if (cc->Inputs().HasTag(kImageFrameTag)) {
cc->Inputs().Tag(kImageFrameTag).Set<mediapipe::ImageFrame>();
}
if (gpu_input) {
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#endif // !MEDIAPIPE_DISABLE_GPU
}
return absl::OkStatus();
}
absl::Status ToImageCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kGpuBufferTag)) {
gpu_input_ = true;
}
if (gpu_input_) {
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
#endif
} // !MEDIAPIPE_DISABLE_GPU
return absl::OkStatus();
}
absl::Status ToImageCalculator::Process(CalculatorContext* cc) {
if (gpu_input_) {
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([&cc]() -> absl::Status {
auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
// Wrap texture pointer; shallow copy.
auto output = std::make_unique<mediapipe::Image>(input);
cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp());
return absl::OkStatus();
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
// The input ImageFrame.
auto& input = cc->Inputs().Tag(kImageFrameTag).Get<mediapipe::ImageFrame>();
// Make a copy of the input packet to co-own the input ImageFrame.
Packet* packet_copy_ptr =
new Packet(cc->Inputs().Tag(kImageFrameTag).Value());
// Create an output Image that (co-)owns a new ImageFrame that points to
// the same pixel data as the input ImageFrame and also owns the packet
// copy. As a result, the output Image indirectly co-owns the input
// ImageFrame. This ensures a correct life span of the shared pixel data.
std::unique_ptr<mediapipe::Image> output =
std::make_unique<mediapipe::Image>(
std::make_shared<mediapipe::ImageFrame>(
input.Format(), input.Width(), input.Height(),
input.WidthStep(), const_cast<uint8*>(input.PixelData()),
[packet_copy_ptr](uint8*) { delete packet_copy_ptr; }));
cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp());
}
ASSIGN_OR_RETURN(auto output, GetInputImage(cc));
kOut(cc).Send(output.At(cc->InputTimestamp()));
return absl::OkStatus();
}
@@ -157,4 +95,43 @@ absl::Status ToImageCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
// Wrap ImageFrameSharedPtr; shallow copy.
absl::StatusOr<Packet<Image>> FromImageFrame(Packet<ImageFrame> packet) {
return MakePacket<Image, std::shared_ptr<mediapipe::ImageFrame>>(
std::const_pointer_cast<mediapipe::ImageFrame>(
SharedPtrWithPacket<mediapipe::ImageFrame>(packet)));
}
// Wrap texture pointer; shallow copy.
absl::StatusOr<Packet<Image>> FromGpuBuffer(Packet<GpuBuffer> packet) {
#if !MEDIAPIPE_DISABLE_GPU
const GpuBuffer& buffer = *packet;
return MakePacket<Image, const GpuBuffer&>(buffer);
#else
return absl::UnimplementedError("GPU processing is disabled in build flags");
#endif // !MEDIAPIPE_DISABLE_GPU
}
absl::StatusOr<Packet<Image>> ToImageCalculator::GetInputImage(
CalculatorContext* cc) {
if (kIn(cc).IsConnected()) {
return kIn(cc).Visit(
[&](const mediapipe::Image&) {
return absl::StatusOr<Packet<Image>>(kIn(cc).As<Image>());
},
[&](const mediapipe::ImageFrame&) {
return FromImageFrame(kIn(cc).As<ImageFrame>());
},
[&](const GpuBuffer&) {
return FromGpuBuffer(kIn(cc).As<GpuBuffer>());
});
} else if (kInCpu(cc).IsConnected()) {
return FromImageFrame(kInCpu(cc).As<ImageFrame>());
} else if (kInGpu(cc).IsConnected()) {
return FromGpuBuffer(kInGpu(cc).As<GpuBuffer>());
}
return absl::InvalidArgumentError("No input found.");
}
} // namespace api2
} // namespace mediapipe