talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
PiperOrigin-RevId: 267274408
This commit is contained in:
MediaPipe Team
2019-09-04 19:00:29 -07:00
committed by jqtang
parent 731d2b9536
commit af67642055
80 changed files with 3181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/README.md
+3
View File
@@ -46,6 +46,7 @@ bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflo
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
--alsologtostderr
```
<<<<<<< HEAD
**TFlite Face Detection**
@@ -97,3 +98,5 @@ bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \
--alsologtostderr
```
=======
>>>>>>> Project import generated by Copybara.
mediapipe/examples/desktop/youtube8m/BUILD
+35
View File
@@ -0,0 +1,35 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
cc_binary(
name = "extract_yt8m_features",
srcs = ["extract_yt8m_features.cc"],
deps = [
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/graphs/youtube8m:yt8m_calculators_deps",
# TODO: Figure out the minimum set of the kernels needed by this example.
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
],
)
mediapipe/examples/desktop/youtube8m/README.md
+54
View File
@@ -0,0 +1,54 @@
### Steps to run the YouTube-8M feature extraction graph
1. Checkout the mediapipe repository
```bash
git clone https://github.com/google/mediapipe.git
cd mediapipe
```
2. Download the PCA and model data
```bash
mkdir /tmp/mediapipe
cd /tmp/mediapipe
curl -O http://data.yt8m.org/pca_matrix_data/inception3_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/inception3_projection_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_projection_matrix_data.pb
curl -O http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz
tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz
```
3. Get the VGGish frozen graph
Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed
with the TensorFlow 1.14+ package installed.
```bash
# cd to the root directory of the MediaPipe repo
cd -
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
```
4. Generate a MediaSequence metadata from the input video
Note: the output file is /tmp/mediapipe/metadata.tfrecord
```bash
python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
--path_to_input_video=/absolute/path/to/the/local/video/file
```
5. Run the MediaPipe binary to extract the features
```bash
bazel build -c opt \
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
./bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features
--calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \
--input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.tfrecord \
--output_side_packets=output_sequence_example=/tmp/mediapipe/output.tfrecord
```
mediapipe/examples/desktop/youtube8m/__init__.py
+14
View File
@@ -0,0 +1,14 @@
"""Copyright 2019 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
+135
View File
@@ -0,0 +1,135 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A simple main function to run a MediaPipe graph. Input side packets are read
// from files provided via the command line and output side packets are written
// to disk.
#include "absl/strings/str_split.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
DEFINE_string(output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
::mediapipe::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
std::map<std::string, ::mediapipe::Packet> input_side_packets;
std::vector<std::string> kv_pairs =
absl::StrSplit(FLAGS_input_side_packets, ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!::mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
std::string input_side_packet_contents;
RETURN_IF_ERROR(mediapipe::file::GetContents(name_and_value[1],
&input_side_packet_contents));
input_side_packets[name_and_value[0]] =
::mediapipe::MakePacket<std::string>(input_side_packet_contents);
}
mediapipe::MatrixData inc3_pca_mean_matrix_data,
inc3_pca_projection_matrix_data, vggish_pca_mean_matrix_data,
vggish_pca_projection_matrix_data;
mediapipe::Matrix inc3_pca_mean_matrix, inc3_pca_projection_matrix,
vggish_pca_mean_matrix, vggish_pca_projection_matrix;
std::string content;
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/inception3_mean_matrix_data.pb", &content));
inc3_pca_mean_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(inc3_pca_mean_matrix_data,
&inc3_pca_mean_matrix);
input_side_packets["inception3_pca_mean_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(inc3_pca_mean_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/inception3_projection_matrix_data.pb", &content));
inc3_pca_projection_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(inc3_pca_projection_matrix_data,
&inc3_pca_projection_matrix);
input_side_packets["inception3_pca_projection_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(inc3_pca_projection_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/vggish_mean_matrix_data.pb", &content));
vggish_pca_mean_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(vggish_pca_mean_matrix_data,
&vggish_pca_mean_matrix);
input_side_packets["vggish_pca_mean_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(vggish_pca_mean_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/vggish_projection_matrix_data.pb", &content));
vggish_pca_projection_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(vggish_pca_projection_matrix_data,
&vggish_pca_projection_matrix);
input_side_packets["vggish_pca_projection_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(vggish_pca_projection_matrix);
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
LOG(INFO) << "Start running the calculator graph.";
RETURN_IF_ERROR(graph.Run());
LOG(INFO) << "Gathering output side packets.";
kv_pairs = absl::StrSplit(FLAGS_output_side_packets, ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
::mediapipe::StatusOr<::mediapipe::Packet> output_packet =
graph.GetOutputSidePacket(name_and_value[0]);
RET_CHECK(output_packet.ok())
<< "Packet " << name_and_value[0] << " was not available.";
const std::string& serialized_string =
output_packet.ValueOrDie().Get<std::string>();
RETURN_IF_ERROR(
mediapipe::file::SetContents(name_and_value[1], serialized_string));
}
return ::mediapipe::OkStatus();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
::mediapipe::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
} else {
LOG(INFO) << "Success!";
}
return 0;
}
mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py
+56
View File
@@ -0,0 +1,56 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate a MediaSequence metadata for MediaPipe input."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
from absl import app
from absl import flags
import tensorflow as tf
from mediapipe.util.sequence import media_sequence as ms
FLAGS = flags.FLAGS
SECONDS_TO_MICROSECONDS = 1000000
def bytes23(string):
"""Creates a bytes string in either Python 2 or 3."""
if sys.version_info >= (3, 0):
return bytes(string, 'utf8')
else:
return bytes(string)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
if not flags.FLAGS.path_to_input_video:
raise ValueError('You must specify the path to the input video.')
metadata = tf.train.SequenceExample()
ms.set_clip_data_path(bytes23(flags.FLAGS.path_to_input_video), metadata)
ms.set_clip_start_timestamp(0, metadata)
ms.set_clip_end_timestamp(
int(float(300 * SECONDS_TO_MICROSECONDS)), metadata)
with open('/tmp/mediapipe/metadata.tfrecord', 'wb') as writer:
writer.write(metadata.SerializeToString())
if __name__ == '__main__':
flags.DEFINE_string('path_to_input_video', '', 'Path to the input video.')
app.run(main)
mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py
+70
View File
@@ -0,0 +1,70 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Code to clone the github repository, download the checkpoint and generate the frozen graph.
The frozen VGGish checkpoint will be saved to `/tmp/mediapipe/vggish_new.pb`.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
from absl import app
import tensorflow as tf
from tensorflow.python.tools import freeze_graph
BASE_DIR = '/tmp/mediapipe/'
def create_vggish_frozen_graph():
"""Create the VGGish frozen graph."""
os.system('git clone https://github.com/tensorflow/models.git')
sys.path.append('models/research/audioset/vggish/')
import vggish_slim
os.system('curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt')
ckpt_path = 'vggish_model.ckpt'
with tf.Graph().as_default(), tf.Session() as sess:
vggish_slim.define_vggish_slim(training=False)
vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path)
saver = tf.train.Saver(tf.all_variables())
freeze_graph.freeze_graph_with_def_protos(
sess.graph_def,
saver.as_saver_def(),
ckpt_path,
'vggish/fc2/BiasAdd',
restore_op_name=None,
filename_tensor_name=None,
output_graph='/tmp/mediapipe/vggish_new.pb',
clear_devices=True,
initializer_nodes=None)
os.system('rm -rf models/')
os.system('rm %s' % ckpt_path)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
create_vggish_frozen_graph()
if __name__ == '__main__':
app.run(main)