Project import generated by Copybara.
PiperOrigin-RevId: 267274408
This commit is contained in:
MediaPipe Team
@@ -0,0 +1,46 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "yt8m_calculators_deps",
|
||||
deps = [
|
||||
"//mediapipe/calculators/audio:audio_decoder_calculator",
|
||||
"//mediapipe/calculators/audio:basic_time_series_calculators",
|
||||
"//mediapipe/calculators/audio:mfcc_mel_calculators",
|
||||
"//mediapipe/calculators/audio:rational_factor_resample_calculator",
|
||||
"//mediapipe/calculators/audio:spectrogram_calculator",
|
||||
"//mediapipe/calculators/audio:stabilized_log_calculator",
|
||||
"//mediapipe/calculators/audio:time_series_framer_calculator",
|
||||
"//mediapipe/calculators/core:add_header_calculator",
|
||||
"//mediapipe/calculators/core:matrix_multiply_calculator",
|
||||
"//mediapipe/calculators/core:matrix_subtract_calculator",
|
||||
"//mediapipe/calculators/core:matrix_to_vector_calculator",
|
||||
"//mediapipe/calculators/core:packet_cloner_calculator",
|
||||
"//mediapipe/calculators/core:packet_resampler_calculator",
|
||||
"//mediapipe/calculators/tensorflow:image_frame_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator",
|
||||
"//mediapipe/calculators/tensorflow:string_to_sequence_example_calculator",
|
||||
"//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator",
|
||||
"//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator",
|
||||
"//mediapipe/calculators/tensorflow:tensorflow_inference_calculator",
|
||||
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator",
|
||||
"//mediapipe/calculators/tensorflow:unpack_media_sequence_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_decoder_calculator",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,295 @@
|
||||
input_side_packet: "input_sequence_example"
|
||||
input_side_packet: "inception3_pca_mean_matrix"
|
||||
input_side_packet: "inception3_pca_projection_matrix"
|
||||
input_side_packet: "vggish_pca_mean_matrix"
|
||||
input_side_packet: "vggish_pca_projection_matrix"
|
||||
output_side_packet: "sequence_example_to_serialize"
|
||||
|
||||
node {
|
||||
calculator: "StringToSequenceExampleCalculator"
|
||||
input_side_packet: "STRING:input_sequence_example"
|
||||
output_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "UnpackMediaSequenceCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
output_side_packet: "DATA_PATH:input_file"
|
||||
output_side_packet: "RESAMPLER_OPTIONS:packet_resampler_options"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.UnpackMediaSequenceCalculatorOptions]: {
|
||||
base_packet_resampler_options {
|
||||
frame_rate: 1.0
|
||||
base_timestamp: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decode the entire video.
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file"
|
||||
output_stream: "VIDEO:decoded_frames"
|
||||
}
|
||||
|
||||
# Extract the subset of frames we want to keep.
|
||||
node {
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "decoded_frames"
|
||||
output_stream: "sampled_decoded_frames"
|
||||
input_side_packet: "OPTIONS:packet_resampler_options"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ImageFrameToTensorCalculator"
|
||||
input_stream: "sampled_decoded_frames"
|
||||
output_stream: "tensor_frame"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
|
||||
output_side_packet: "SESSION:session"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions]: {
|
||||
graph_proto_path: "/tmp/mediapipe/classify_image_graph_def.pb"
|
||||
tag_to_tensor_names {
|
||||
key: "IMG_UINT8"
|
||||
value: "DecodeJpeg:0"
|
||||
}
|
||||
tag_to_tensor_names {
|
||||
key: "INCEPTION_POOL3"
|
||||
value: "pool_3/_reshape:0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorFlowInferenceCalculator"
|
||||
input_side_packet: "SESSION:session"
|
||||
input_stream: "IMG_UINT8:tensor_frame"
|
||||
output_stream: "INCEPTION_POOL3:inception3_hidden_activation_single_element_batch"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorFlowInferenceCalculatorOptions]: {
|
||||
signature_name: ""
|
||||
batch_size: 1
|
||||
add_batch_dim_to_tensors: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Remove the batch dimension.
|
||||
node: {
|
||||
calculator: "TensorSqueezeDimensionsCalculator"
|
||||
input_stream: "inception3_hidden_activation_single_element_batch"
|
||||
output_stream: "inception3_hidden_activation"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorSqueezeDimensionsCalculatorOptions]: {
|
||||
dim: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorToMatrixCalculator"
|
||||
input_stream: "TENSOR:inception3_hidden_activation"
|
||||
output_stream: "MATRIX:inception3_hidden_activation_matrix"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "MINUEND:inception3_hidden_activation_matrix"
|
||||
input_side_packet: "SUBTRAHEND:inception3_pca_mean_matrix"
|
||||
output_stream: "mean_subtracted_inception3_matrix"
|
||||
}
|
||||
node {
|
||||
calculator: "MatrixMultiplyCalculator"
|
||||
input_stream: "mean_subtracted_inception3_matrix"
|
||||
input_side_packet: "inception3_pca_projection_matrix"
|
||||
output_stream: "pca_inception3_matrix"
|
||||
}
|
||||
node {
|
||||
calculator: "MatrixToVectorCalculator"
|
||||
input_stream: "pca_inception3_matrix"
|
||||
output_stream: "pca_inception3_vf"
|
||||
}
|
||||
|
||||
######################## END OF VISUAL ###########################
|
||||
|
||||
######################## BEGIN OF AUDIO ##########################
|
||||
node {
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file"
|
||||
output_stream: "AUDIO:audio"
|
||||
output_stream: "AUDIO_HEADER:audio_header"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "AddHeaderCalculator"
|
||||
input_stream: "DATA:audio"
|
||||
input_stream: "HEADER:audio_header"
|
||||
output_stream: "media_audio"
|
||||
}
|
||||
|
||||
# Always convert the audio to mono.
|
||||
node {
|
||||
calculator: "AverageTimeSeriesAcrossChannelsCalculator"
|
||||
input_stream: "media_audio"
|
||||
output_stream: "mono_waveform"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "RationalFactorResampleCalculator"
|
||||
input_stream: "mono_waveform"
|
||||
output_stream: "resampled_waveform"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RationalFactorResampleCalculatorOptions] {
|
||||
target_sample_rate: 16000.0
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "SpectrogramCalculator"
|
||||
input_stream: "resampled_waveform"
|
||||
output_stream: "spectrogram_squared_magnitude"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SpectrogramCalculatorOptions] {
|
||||
frame_duration_seconds: 0.025
|
||||
frame_overlap_seconds: 0.015
|
||||
output_type: SQUARED_MAGNITUDE
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "MelSpectrumCalculator"
|
||||
# MelSpectrumCalculator expects SQUARED_MAGNITUDE input, but its output is in
|
||||
# linear magnitude units.
|
||||
input_stream: "spectrogram_squared_magnitude"
|
||||
output_stream: "mel_spectrum_magnitude"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.MelSpectrumCalculatorOptions] {
|
||||
# Follow the 'wideband' or '16kHz' speech convention.
|
||||
channel_count: 64
|
||||
min_frequency_hertz: 125.0
|
||||
max_frequency_hertz: 7500.0
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "StabilizedLogCalculator"
|
||||
input_stream: "mel_spectrum_magnitude"
|
||||
output_stream: "log_mel_spectrum_magnitude"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.StabilizedLogCalculatorOptions] {
|
||||
stabilizer: 0.01
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "TimeSeriesFramerCalculator"
|
||||
input_stream: "log_mel_spectrum_magnitude"
|
||||
output_stream: "log_mel_spectrum_magnitude_with_context"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TimeSeriesFramerCalculatorOptions] {
|
||||
frame_duration_seconds: 0.96
|
||||
frame_overlap_seconds: -0.04
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "MatrixToTensorCalculator"
|
||||
input_stream: "log_mel_spectrum_magnitude_with_context"
|
||||
output_stream: "log_mel_spectrum_magnitude_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.MatrixToTensorCalculatorOptions] {
|
||||
transpose: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
|
||||
output_side_packet: "SESSION:vggish_session"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions]: {
|
||||
graph_proto_path: "/tmp/mediapipe/vggish_new.pb"
|
||||
tag_to_tensor_names {
|
||||
key: "INPUT"
|
||||
value: "vggish/input_features:0"
|
||||
}
|
||||
tag_to_tensor_names {
|
||||
key: "VGGISH"
|
||||
value: "vggish/fc2/BiasAdd:0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorFlowInferenceCalculator"
|
||||
input_side_packet: "SESSION:vggish_session"
|
||||
input_stream: "INPUT:log_mel_spectrum_magnitude_tensor"
|
||||
output_stream: "VGGISH:vggish_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorFlowInferenceCalculatorOptions]: {
|
||||
signature_name: ""
|
||||
batch_size: 128
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TensorToMatrixCalculator"
|
||||
input_stream: "REFERENCE:log_mel_spectrum_magnitude_with_context"
|
||||
input_stream: "TENSOR:vggish_tensor"
|
||||
output_stream: "MATRIX:vggish_matrix"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TensorToMatrixCalculatorOptions] {
|
||||
time_series_header_overrides {
|
||||
num_channels: 128
|
||||
num_samples: 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "MINUEND:vggish_matrix"
|
||||
input_side_packet: "SUBTRAHEND:vggish_pca_mean_matrix"
|
||||
output_stream: "mean_subtracted_vggish_matrix"
|
||||
}
|
||||
node {
|
||||
calculator: "MatrixMultiplyCalculator"
|
||||
input_stream: "mean_subtracted_vggish_matrix"
|
||||
input_side_packet: "vggish_pca_projection_matrix"
|
||||
output_stream: "pca_vggish_matrix"
|
||||
}
|
||||
node {
|
||||
calculator: "MatrixToVectorCalculator"
|
||||
input_stream: "pca_vggish_matrix"
|
||||
output_stream: "pca_vggish_vf"
|
||||
}
|
||||
|
||||
# Store the features in the SequenceExample.
|
||||
node {
|
||||
calculator: "PackMediaSequenceCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
output_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
|
||||
input_stream: "FLOAT_FEATURE_RGB:pca_inception3_vf"
|
||||
input_stream: "FLOAT_FEATURE_AUDIO:pca_vggish_vf"
|
||||
}
|
||||
|
||||
# Serialize the SequenceExample to a string for storage.
|
||||
node {
|
||||
calculator: "StringToSequenceExampleCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
|
||||
output_side_packet: "STRING:output_sequence_example"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user