Project import generated by Copybara.

GitOrigin-RevId: 796203faee20d7aae2876aac8ca5a1827dee4fe3
This commit is contained in:
MediaPipe Team
2019-09-30 11:26:36 -07:00
committed by jqtang
parent 412ab42d1f
commit a2a63e3876
122 changed files with 7330 additions and 2016 deletions
+44 -1
View File
@@ -14,7 +14,9 @@
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
package(default_visibility = [
"//visibility:public",
])
cc_library(
name = "simple_run_graph_main",
@@ -29,3 +31,44 @@ cc_library(
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "demo_run_graph_main",
srcs = ["demo_run_graph_main.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
],
)
# Linux only.
# Must have a GPU with EGL support:
# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
# (or similar nvidia/amd equivalent)
cc_library(
name = "demo_run_graph_main_gpu",
srcs = ["demo_run_graph_main_gpu.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_shared_data_internal",
],
)
@@ -0,0 +1,146 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An example of sending OpenCV webcam frames into a MediaPipe graph.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
::mediapipe::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
MP_RETURN_IF_ERROR(graph.Initialize(config));
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
if (save_video) {
LOG(INFO) << "Prepare video writer.";
cv::Mat test_frame;
capture.read(test_frame); // Consume first frame.
capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning.
writer.open(FLAGS_output_video_path,
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), test_frame.size());
RET_CHECK(writer.isOpened());
} else {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
}
LOG(INFO) << "Start running the calculator graph.";
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
graph.AddOutputStreamPoller(kOutputStream));
MP_RETURN_IF_ERROR(graph.StartRun({}));
LOG(INFO) << "Start grabbing and processing frames.";
size_t frame_timestamp = 0;
bool grab_frames = true;
while (grab_frames) {
// Capture opencv camera or video frame.
cv::Mat camera_frame_raw;
capture >> camera_frame_raw;
if (camera_frame_raw.empty()) break; // End of video.
cv::Mat camera_frame;
cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
if (!load_video) {
cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
}
// Wrap Mat into an ImageFrame.
auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
mediapipe::ImageFrame::kDefaultAlignmentBoundary);
cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
camera_frame.copyTo(input_frame_mat);
// Send image packet into the graph.
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
kInputStream, mediapipe::Adopt(input_frame.release())
.At(mediapipe::Timestamp(frame_timestamp++))));
// Get the graph result packet, or stop if that fails.
mediapipe::Packet packet;
if (!poller.Next(&packet)) break;
auto& output_frame = packet.Get<mediapipe::ImageFrame>();
// Convert back to opencv for display or saving.
cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
if (save_video) {
writer.write(output_frame_mat);
} else {
cv::imshow(kWindowName, output_frame_mat);
// Press any key to exit.
const int pressed_key = cv::waitKey(5);
if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
}
}
LOG(INFO) << "Shutting down.";
if (writer.isOpened()) writer.release();
MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));
return graph.WaitUntilDone();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
::mediapipe::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
} else {
LOG(INFO) << "Success!";
}
return 0;
}
@@ -0,0 +1,186 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An example of sending OpenCV webcam frames into a MediaPipe graph.
// This example requires a linux computer and a GPU with EGL support drivers.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
::mediapipe::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
MP_RETURN_IF_ERROR(graph.Initialize(config));
LOG(INFO) << "Initialize the GPU.";
ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create());
MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources)));
mediapipe::GlCalculatorHelper gpu_helper;
gpu_helper.InitializeForTest(graph.GetGpuResources().get());
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
if (save_video) {
LOG(INFO) << "Prepare video writer.";
cv::Mat test_frame;
capture.read(test_frame); // Consume first frame.
capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning.
writer.open(FLAGS_output_video_path,
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), test_frame.size());
RET_CHECK(writer.isOpened());
} else {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
}
LOG(INFO) << "Start running the calculator graph.";
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
graph.AddOutputStreamPoller(kOutputStream));
MP_RETURN_IF_ERROR(graph.StartRun({}));
LOG(INFO) << "Start grabbing and processing frames.";
size_t frame_timestamp = 0;
bool grab_frames = true;
while (grab_frames) {
// Capture opencv camera or video frame.
cv::Mat camera_frame_raw;
capture >> camera_frame_raw;
if (camera_frame_raw.empty()) break; // End of video.
cv::Mat camera_frame;
cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
if (!load_video) {
cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
}
// Wrap Mat into an ImageFrame.
auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);
cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
camera_frame.copyTo(input_frame_mat);
// Prepare and add graph input packet.
MP_RETURN_IF_ERROR(
gpu_helper.RunInGlContext([&input_frame, &frame_timestamp, &graph,
&gpu_helper]() -> ::mediapipe::Status {
// Convert ImageFrame to GpuBuffer.
auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
auto gpu_frame = texture.GetFrame<mediapipe::GpuBuffer>();
glFlush();
texture.Release();
// Send GPU image packet into the graph.
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
kInputStream, mediapipe::Adopt(gpu_frame.release())
.At(mediapipe::Timestamp(frame_timestamp++))));
return ::mediapipe::OkStatus();
}));
// Get the graph result packet, or stop if that fails.
mediapipe::Packet packet;
if (!poller.Next(&packet)) break;
std::unique_ptr<mediapipe::ImageFrame> output_frame;
// Convert GpuBuffer to ImageFrame.
MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext(
[&packet, &output_frame, &gpu_helper]() -> ::mediapipe::Status {
auto& gpu_frame = packet.Get<mediapipe::GpuBuffer>();
auto texture = gpu_helper.CreateSourceTexture(gpu_frame);
output_frame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormatForGpuBufferFormat(gpu_frame.format()),
gpu_frame.width(), gpu_frame.height(),
mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);
gpu_helper.BindFramebuffer(texture);
const auto info =
mediapipe::GlTextureInfoForGpuBufferFormat(gpu_frame.format(), 0);
glReadPixels(0, 0, texture.width(), texture.height(), info.gl_format,
info.gl_type, output_frame->MutablePixelData());
glFlush();
texture.Release();
return ::mediapipe::OkStatus();
}));
// Convert back to opencv for display or saving.
cv::Mat output_frame_mat = mediapipe::formats::MatView(output_frame.get());
cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
if (save_video) {
writer.write(output_frame_mat);
} else {
cv::imshow(kWindowName, output_frame_mat);
// Press any key to exit.
const int pressed_key = cv::waitKey(5);
if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
}
}
LOG(INFO) << "Shutting down.";
if (writer.isOpened()) writer.release();
MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));
return graph.WaitUntilDone();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
::mediapipe::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
} else {
LOG(INFO) << "Success!";
}
return 0;
}
@@ -0,0 +1,34 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
cc_binary(
name = "face_detection_cpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/face_detection:desktop_tflite_calculators",
],
)
# Linux only
cc_binary(
name = "face_detection_gpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main_gpu",
"//mediapipe/graphs/face_detection:mobile_calculators",
],
)
@@ -0,0 +1,26 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
# Linux only
cc_binary(
name = "hair_segmentation_gpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main_gpu",
"//mediapipe/graphs/hair_segmentation:mobile_calculators",
],
)
@@ -0,0 +1,42 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
cc_binary(
name = "hand_tracking_tflite",
deps = [
"//mediapipe/examples/desktop:simple_run_graph_main",
"//mediapipe/graphs/hand_tracking:desktop_tflite_calculators",
],
)
cc_binary(
name = "hand_tracking_cpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/hand_tracking:desktop_tflite_calculators",
],
)
# Linux only
cc_binary(
name = "hand_tracking_gpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main_gpu",
"//mediapipe/graphs/hand_tracking:mobile_calculators",
],
)
@@ -72,3 +72,11 @@ cc_binary(
"//mediapipe/graphs/object_detection:desktop_tflite_calculators",
],
)
cc_binary(
name = "object_detection_cpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/object_detection:desktop_tflite_calculators",
],
)
+1 -1
View File
@@ -27,7 +27,7 @@ cc_binary(
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/graphs/youtube8m:yt8m_calculators_deps",
"//mediapipe/graphs/youtube8m:yt8m_feature_extraction_calculators",
# TODO: Figure out the minimum set of the kernels needed by this example.
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
@@ -37,7 +37,9 @@
```bash
python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
--path_to_input_video=/absolute/path/to/the/local/video/file
--path_to_input_video=/absolute/path/to/the/local/video/file \
--clip_start_time_sec=0 \
--clip_end_time_sec=10
```
5. Run the MediaPipe binary to extract the features
@@ -37,20 +37,29 @@ def bytes23(string):
def main(argv):
if len(argv) > 1:
if len(argv) > 3:
raise app.UsageError('Too many command-line arguments.')
if not flags.FLAGS.path_to_input_video:
raise ValueError('You must specify the path to the input video.')
if not flags.FLAGS.clip_end_time_sec:
raise ValueError('You must specify the clip end timestamp in seconds.')
if flags.FLAGS.clip_start_time_sec >= flags.FLAGS.clip_end_time_sec:
raise ValueError(
'The clip start time must be greater than the clip end time.')
metadata = tf.train.SequenceExample()
ms.set_clip_data_path(bytes23(flags.FLAGS.path_to_input_video), metadata)
ms.set_clip_start_timestamp(0, metadata)
ms.set_clip_start_timestamp(
flags.FLAGS.clip_start_time_sec * SECONDS_TO_MICROSECONDS, metadata)
ms.set_clip_end_timestamp(
int(float(300 * SECONDS_TO_MICROSECONDS)), metadata)
flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata)
with open('/tmp/mediapipe/metadata.tfrecord', 'wb') as writer:
writer.write(metadata.SerializeToString())
if __name__ == '__main__':
flags.DEFINE_string('path_to_input_video', '', 'Path to the input video.')
flags.DEFINE_integer('clip_start_time_sec', 0,
'Clip start timestamp in seconds')
flags.DEFINE_integer('clip_end_time_sec', 10, 'Clip end timestamp in seconds')
app.run(main)