Project import generated by Copybara.

GitOrigin-RevId: 1e13be30e2c6838d4a2ff768a39c414bc80534bb
This commit is contained in:
MediaPipe Team
2022-09-06 21:46:17 +00:00
committed by Sebastian Schmidt
parent 63e679d99c
commit 4dc4b19ddb
639 changed files with 71327 additions and 2078 deletions
+1
View File
@@ -253,6 +253,7 @@ cc_library(
deps = [
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/formats:location_opencv",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:variant",
"//mediapipe/framework/port:status",
+48
View File
@@ -24,6 +24,7 @@
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "libyuv/convert.h"
#include "libyuv/convert_argb.h"
#include "libyuv/convert_from.h"
#include "libyuv/row.h"
#include "libyuv/video_common.h"
@@ -162,6 +163,53 @@ void YUVImageToImageFrame(const YUVImage& yuv_image, ImageFrame* image_frame,
CHECK_EQ(0, rv);
}
void YUVImageToImageFrameFromFormat(const YUVImage& yuv_image,
ImageFrame* image_frame) {
CHECK(image_frame);
int width = yuv_image.width();
int height = yuv_image.height();
image_frame->Reset(ImageFormat::SRGB, width, height, 16);
const auto& format = yuv_image.fourcc();
switch (format) {
case libyuv::FOURCC_NV12:
// 8-bit Y plane followed by an interleaved 8-bit U/V plane with 2×2
// subsampling.
libyuv::NV12ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), image_frame->MutablePixelData(),
image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_NV21:
// 8-bit Y plane followed by an interleaved 8-bit V/U plane with 2×2
// subsampling.
libyuv::NV21ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), image_frame->MutablePixelData(),
image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_I420:
// Also known as YV21.
// 8-bit Y plane followed by 8-bit 2×2 subsampled U and V planes.
libyuv::I420ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
yuv_image.stride(1), yuv_image.data(2), yuv_image.stride(2),
image_frame->MutablePixelData(), image_frame->WidthStep(),
yuv_image.width(), yuv_image.height());
break;
case libyuv::FOURCC_YV12:
// 8-bit Y plane followed by 8-bit 2×2 subsampled V and U planes.
libyuv::I420ToRAW(
yuv_image.data(0), yuv_image.stride(0), yuv_image.data(2),
yuv_image.stride(2), yuv_image.data(1), yuv_image.stride(1),
image_frame->MutablePixelData(), image_frame->WidthStep(),
yuv_image.width(), yuv_image.height());
break;
default:
LOG(FATAL) << "Unsupported YUVImage format.";
}
}
void SrgbToMpegYCbCr(const uint8 r, const uint8 g, const uint8 b, //
uint8* y, uint8* cb, uint8* cr) {
// ITU-R BT.601 conversion from sRGB to YCbCr.
+5
View File
@@ -64,6 +64,11 @@ void ImageFrameToYUVNV12Image(const ImageFrame& image_frame,
void YUVImageToImageFrame(const YUVImage& yuv_image, ImageFrame* image_frame,
bool use_bt709 = false);
// Converts a YUV image to an image frame, based on the yuv_image.fourcc()
// format. Fails if no format is provided.
void YUVImageToImageFrameFromFormat(const YUVImage& yuv_image,
ImageFrame* image_frame);
// Convert sRGB values into MPEG YCbCr values. Notice that MPEG YCbCr
// values use a smaller range of values than JPEG YCbCr. The conversion
// values used are those from ITU-R BT.601 (which are the same as ITU-R
+12
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fstream>
#include "absl/flags/flag.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/file_helpers.h"
@@ -37,6 +39,16 @@ absl::Status DefaultGetResourceContents(const std::string& path,
} // namespace internal
absl::StatusOr<std::string> PathToResourceAsFile(const std::string& path) {
if (absl::StartsWith(path, "/")) {
return path;
}
// Try to load the file from bazel-bin. If it does not exist, fall back to the
// resource folder.
auto bazel_path = JoinPath("bazel-bin", path);
if (file::Exists(bazel_path).ok()) {
return bazel_path;
}
return JoinPath(absl::GetFlag(FLAGS_resource_root_dir), path);
}
+57 -25
View File
@@ -22,14 +22,17 @@ number of features per timestep varies, creating a ragged struction. Video
object detection is one example task that requires this ragged structure because
the number of detections per frame varies. SequenceExamples can easily encode
this ragged structure. Sequences naturally match the semantics of video as a
sequence of frames or other common media patterns. The interpretable semantics simplify debugging and decoding of
potentially complicated data. One potential disadvantage of SequenceExamples is
that keys and formats can vary widely. The MediaSequence library provides tools
for consistently manipulating and decoding SequenceExamples in Python and C++ in
a consistent format. The consistent format enables creating a pipeline for
processing data sets. A goal of MediaSequence as a pipeline is that users should
only need to specify the metadata (e.g. videos and labels) for their task. The
pipeline will turn the metadata into training data.
sequence of frames or other common media patterns. The video feature lists will
be stored in order with strictly increasing timestamps so the data is
unambiguously ordered. The interpretable semantics simplify debugging and
decoding of potentially complicated data. One potential disadvantage of
SequenceExamples is that keys and formats can vary widely. The MediaSequence
library provides tools for consistently manipulating and decoding
SequenceExamples in Python and C++ in a consistent format. The consistent format
enables creating a pipeline for processing data sets. A goal of MediaSequence as
a pipeline is that users should only need to specify the metadata (e.g. videos
and labels) for their task. The pipeline will turn the metadata into training
data.
The pipeline has two stages. First, users must generate the metadata
describing the data and applicable labels. This process is
@@ -233,8 +236,10 @@ The values that are unpacked and packed into these calculators are determined
by the tags on the streams in the MediaPipe calculator graph. (Tags are required
to be all capitals and underscores. To encode prefixes for feature keys as tags,
prefixes for feature keys should follow the same convention.) The documentation
for these two calculators describes the variety of data they support. Any other
MediaPipe processing can be used between these calculators to extract features.
for these two calculators describes the variety of data they support. The
timestamps of each feature list being unpacked must be in strictly increasing
order. Any other MediaPipe processing can be used between these calculators to
extract features.
#### Adding data and reconciling metadata
In general, the pipeline will decode the specified media between the clip
@@ -279,6 +284,31 @@ optical flow can't be estimated for the last frame of a video clip, so it
adds one less frame of data. With the exception of aligning bounding boxes, the
pipeline does nothing to require consistent timestamps between features.
### Using prefixes
Prefixes enable storing semantically identical data without collisions. For
example, it is possible to store predicted and ground truth bounding boxes by
using different prefixes. We can also store bounding boxes and labels from
different tasks by utilizing prefixes.
To minimize burdening the API and documentation, eschew using prefixes unless
necessary.
The recommended prefix format, enforced by some MediaPipe functions, is all caps
with underscores, and numeric characters after the first character. e.g.
`MY_FAVORITE_FEATURE_V1`.
The convention for encoding groundtruth labels is to use no prefix, while
predicted labels are typically tagged with prefixes. For example:
* Example groudntruth keys:
* `region/label/string`
* `region/label/confidence`
* Example predicted label keys:
* `PREDICT_V1/region/label/string`
* `PREDICT_V1/region/label/confidence`
## Function prototypes for each data type
MediaSequence provides accessors to store common data patterns in
@@ -288,14 +318,11 @@ the key, so we will document the functions with a generic name, Feature. Note
that due to different conventions for Python and C++ code, the capitalization
and parameter order varies, but the functionality should be equivalent.
Each function takes an optional prefix parameter. Prefixes enable storing
semantically identical data without collisions. For example, it is possible to
store predicted and ground truth bounding boxes by using different prefixes.
To minimize burdening the API and documentation, eschew using prefixes unless
necessary. For some common cases, such as storing instance segmentation labels
along with images, named versions with prefixes baked in provided as documented
below. Lastly, generic features and audio streams should almost always use a
prefix because storing multiple features or transformed audio streams is common.
Each function takes an optional prefix parameter. For some common cases, such as
storing instance segmentation labels along with images, named versions with
prefixes baked in provided as documented below. Lastly, generic features and
audio streams should almost always use a prefix because storing multiple
features or transformed audio streams is common.
The code generating these functions resides in media_sequence.h/.cc/.py and
media_sequence_util.h/.cc/.py. The media_sequence files generally defines the
@@ -368,6 +395,7 @@ are provided for elaboration.
|-----|------|------------------------|-------------|
|`example/id`|context bytes|`set_example_id` / `SetExampleId`|A unique identifier for each example.|
|`example/dataset_name`|context bytes|`set_example_dataset_name` / `SetExampleDatasetName`|The name of the data set, including the version.|
|`example/dataset/flag/string`|context bytes list|`set_example_dataset_flag_string` / `SetExampleDatasetFlagString`|A list of bytes for dataset related attributes or flags for this example.
### Keys related to a clip
| key | type | python call / c++ call | description |
@@ -381,7 +409,7 @@ are provided for elaboration.
|`clip/media_id`|context bytes|`set_clip_media_id` / `SetClipMediaId`|Any identifier for the media beyond the data path.|
|`clip/alternative_media_id`|context bytes|`set_clip_alternative_media_id` / `SetClipAlternativeMediaId`|Yet another alternative identifier.|
|`clip/encoded_media_bytes`|context bytes|`set_clip_encoded_media_bytes` / `SetClipEncodedMediaBytes`|The encoded bytes for storing media directly in the SequenceExample.|
|`clip/ encoded_media_start_timestamp`|context int|`set_clip_encoded_media_start_timestamp` / `SetClipEncodedMediaStartTimestamp`|The start time for the encoded media if not preserved during encoding.
|`clip/encoded_media_start_timestamp`|context int|`set_clip_encoded_media_start_timestamp` / `SetClipEncodedMediaStartTimestamp`|The start time for the encoded media if not preserved during encoding.|
### Keys related to segments of clips
| key | type | python call / c++ call | description |
@@ -447,7 +475,7 @@ tasks and tracking (or class) fields for tracking information.
|`region/embedding/format`|context string|`set_bbox_embedding_format` / `SetBBoxEmbeddingFormat`|Provides the encoding format, if any, for region embeddings.|
|`region/embedding/encoded`|feature list bytes list|`add_bbox_embedding_encoded` / `AddBBoxEmbeddingEncoded`|For each region, provide an encoded embedding.|
|`region/embedding/confidence`|feature list float list|`add_bbox_embedding_confidence` / `AddBBoxEmbeddingConfidence` | For each region, provide a confidence for the embedding.|
|`region/unmodified_timestamp`|feature list int|`add_bbox_unmodified_timestamp` / `AddBBoxUnmodifiedTimestamp`|Used to store the original timestamps if procedurally aligning timestamps to image frames.|
|`region/unmodified_timestamp`|feature list int|`add_bbox_unmodified_timestamp` / `AddUnmodifiedBBoxTimestamp`|Used to store the original timestamps if procedurally aligning timestamps to image frames.|
### Keys related to images
| key | type | python call / c++ call | description |
@@ -511,7 +539,10 @@ recommendation is to use more specific methods if possible. When using these
generic features, always supply a prefix. (The recommended prefix format,
enforced by some MediaPipe functions, is all caps with underscores, e.g.
MY_FAVORITE_FEATURE.) Following this recommendation, the keys will be listed
with a generic PREFIX.
with a generic PREFIX. Calls exist for storing generic features in both the
`feature_list` and the `context`. For anything that occurs with a timestamp,
use the `feature_list`; for anything that applies to the example as a whole,
without timestamps, use the `context`.
| key | type | python call / c++ call | description |
|-----|------|------------------------|-------------|
@@ -524,13 +555,14 @@ with a generic PREFIX.
|`PREFIX/feature/dimensions`|context int list|`set_feature_dimensions` / `SetFeatureDimensions`|A list of integer dimensions for each feature.|
|`PREFIX/feature/rate`|context float|`set_feature_rate` / `SetFeatureRate`|The rate that features are calculated as features per second.|
|`PREFIX/feature/bytes/format`|context bytes|`set_feature_bytes_format` / `SetFeatureBytesFormat`|The encoding format if any for features stored as bytes.|
|`PREFIX/context_feature/floats`|context float list|`add_context_feature_floats` / `AddContextFeatureFloats`|A list of floats for the entire example.|
|`PREFIX/context_feature/bytes`|context bytes list|`add_context_feature_bytes` / `AddContextFeatureBytes`|A list of bytes for the entire example. Maybe be encoded.|
|`PREFIX/context_feature/ints`|context int list|`add_context_feature_ints` / `AddContextFeatureInts`|A list of ints for the entire example.|
### Keys related to audio
Audio is a special subtype of generic features with additional data about the
audio format. When using audio, always supply a prefix. (The recommended prefix
format, enforced by some MediaPipe functions, is all caps with underscores, e.g.
MY_FAVORITE_FEATURE.) Following this recommendation, the keys will be listed
with a generic PREFIX.
audio format. When using audio, always supply a prefix. The keys here will be
listed with a generic PREFIX.
To understand the terminology, it is helpful conceptualize the audio as a list
of matrices. The columns of the matrix are called samples. The rows of the
+12
View File
@@ -179,6 +179,8 @@ namespace mediasequence {
const char kExampleIdKey[] = "example/id";
// The name of the data set, including the version.
const char kExampleDatasetNameKey[] = "example/dataset_name";
// String flags or attributes for this example within a data set.
const char kExampleDatasetFlagStringKey[] = "example/dataset/flag/string";
// The relative path to the data on disk from some root directory.
const char kClipDataPathKey[] = "clip/data_path";
@@ -204,6 +206,8 @@ const char kClipLabelConfidenceKey[] = "clip/label/confidence";
BYTES_CONTEXT_FEATURE(ExampleId, kExampleIdKey);
BYTES_CONTEXT_FEATURE(ExampleDatasetName, kExampleDatasetNameKey);
VECTOR_BYTES_CONTEXT_FEATURE(ExampleDatasetFlagString,
kExampleDatasetFlagStringKey);
BYTES_CONTEXT_FEATURE(ClipDataPath, kClipDataPathKey);
BYTES_CONTEXT_FEATURE(ClipAlternativeMediaId, kClipAlternativeMediaId);
@@ -667,6 +671,10 @@ const char kFeaturePacketRateKey[] = "feature/packet_rate";
const char kFeatureAudioSampleRateKey[] = "feature/audio_sample_rate";
// The feature as a list of floats.
const char kContextFeatureFloatsKey[] = "context_feature/floats";
// The feature as a list of floats.
const char kContextFeatureBytesKey[] = "context_feature/bytes";
// The feature as a list of floats.
const char kContextFeatureIntsKey[] = "context_feature/ints";
// Feature list keys:
// The feature as a list of floats.
@@ -699,6 +707,10 @@ PREFIXED_VECTOR_INT64_CONTEXT_FEATURE(FeatureDimensions, kFeatureDimensionsKey);
PREFIXED_FLOAT_CONTEXT_FEATURE(FeatureRate, kFeatureRateKey);
PREFIXED_VECTOR_FLOAT_CONTEXT_FEATURE(ContextFeatureFloats,
kContextFeatureFloatsKey);
PREFIXED_VECTOR_BYTES_CONTEXT_FEATURE(ContextFeatureBytes,
kContextFeatureBytesKey);
PREFIXED_VECTOR_INT64_CONTEXT_FEATURE(ContextFeatureInts,
kContextFeatureIntsKey);
PREFIXED_BYTES_CONTEXT_FEATURE(FeatureBytesFormat, kFeatureBytesFormatKey);
PREFIXED_VECTOR_FLOAT_FEATURE_LIST(FeatureFloats, kFeatureFloatsKey);
PREFIXED_VECTOR_BYTES_FEATURE_LIST(FeatureBytes, kFeatureBytesKey);
+17
View File
@@ -166,6 +166,8 @@ _HAS_DYNAMIC_ATTRIBUTES = True
EXAMPLE_ID_KEY = "example/id"
# The name o fthe data set, including the version.
EXAMPLE_DATASET_NAME_KEY = "example/dataset_name"
# String flags or attributes for this example within a data set.
EXAMPLE_DATASET_FLAG_STRING_KEY = "example/dataset/flag/string"
# The relative path to the data on disk from some root directory.
CLIP_DATA_PATH_KEY = "clip/data_path"
# Any identifier for the media beyond the data path.
@@ -190,6 +192,9 @@ msu.create_bytes_context_feature(
"example_id", EXAMPLE_ID_KEY, module_dict=globals())
msu.create_bytes_context_feature(
"example_dataset_name", EXAMPLE_DATASET_NAME_KEY, module_dict=globals())
msu.create_bytes_list_context_feature(
"example_dataset_flag_string", EXAMPLE_DATASET_FLAG_STRING_KEY,
module_dict=globals())
msu.create_bytes_context_feature(
"clip_media_id", CLIP_MEDIA_ID_KEY, module_dict=globals())
msu.create_bytes_context_feature(
@@ -646,6 +651,12 @@ FEATURE_TIMESTAMP_KEY = "feature/timestamp"
FEATURE_DURATION_KEY = "feature/duration"
# Encodes an optional confidence score for the generated features.
FEATURE_CONFIDENCE_KEY = "feature/confidence"
# The feature as a list of floats in the context.
CONTEXT_FEATURE_FLOATS_KEY = "context_feature/floats"
# The feature as a list of bytes in the context. May be encoded.
CONTEXT_FEATURE_BYTES_KEY = "context_feature/bytes"
# The feature as a list of ints in the context.
CONTEXT_FEATURE_INTS_KEY = "context_feature/ints"
msu.create_int_list_context_feature(
"feature_dimensions", FEATURE_DIMENSIONS_KEY, module_dict=globals())
@@ -676,4 +687,10 @@ msu.create_int_list_feature_list(
"feature_duration", FEATURE_DURATION_KEY, module_dict=globals())
msu.create_float_list_feature_list(
"feature_confidence", FEATURE_CONFIDENCE_KEY, module_dict=globals())
msu.create_float_list_context_feature(
"context_feature_floats", CONTEXT_FEATURE_FLOATS_KEY, module_dict=globals())
msu.create_bytes_list_context_feature(
"context_feature_bytes", CONTEXT_FEATURE_BYTES_KEY, module_dict=globals())
msu.create_int_list_context_feature(
"context_feature_ints", CONTEXT_FEATURE_INTS_KEY, module_dict=globals())
@@ -35,6 +35,14 @@ TEST(MediaSequenceTest, RoundTripDatasetName) {
ASSERT_EQ(GetExampleDatasetName(sequence), name);
}
TEST(MediaSequenceTest, RoundTripDatasetFlagString) {
tensorflow::SequenceExample sequence;
std::vector<std::string> flags = {"test", "overall", "special"};
SetExampleDatasetFlagString(flags, &sequence);
ASSERT_THAT(GetExampleDatasetFlagString(sequence),
testing::ElementsAreArray(flags));
}
TEST(MediaSequenceTest, RoundTripMediaId) {
tensorflow::SequenceExample sequence;
std::string id = "test";
@@ -625,6 +633,39 @@ TEST(MediaSequenceTest, RoundTripFeatureTimestamp) {
ASSERT_EQ(GetFeatureTimestampSize(feature_key, sequence), 0);
}
TEST(MediaSequenceTest, RoundTripContextFeatureFloats) {
tensorflow::SequenceExample sequence;
std::string feature_key = "TEST";
std::vector<float> vf = {0., 1., 2., 4.};
SetContextFeatureFloats(feature_key, vf, &sequence);
ASSERT_EQ(GetContextFeatureFloats(feature_key, sequence).size(), vf.size());
ASSERT_EQ(GetContextFeatureFloats(feature_key, sequence)[3], vf[3]);
ClearContextFeatureFloats(feature_key, &sequence);
ASSERT_FALSE(HasFeatureFloats(feature_key, sequence));
}
TEST(MediaSequenceTest, RoundTripContextFeatureBytes) {
tensorflow::SequenceExample sequence;
std::string feature_key = "TEST";
std::vector<std::string> vs = {"0", "1", "2", "4"};
SetContextFeatureBytes(feature_key, vs, &sequence);
ASSERT_EQ(GetContextFeatureBytes(feature_key, sequence).size(), vs.size());
ASSERT_EQ(GetContextFeatureBytes(feature_key, sequence)[3], vs[3]);
ClearContextFeatureBytes(feature_key, &sequence);
ASSERT_FALSE(HasFeatureBytes(feature_key, sequence));
}
TEST(MediaSequenceTest, RoundTripContextFeatureInts) {
tensorflow::SequenceExample sequence;
std::string feature_key = "TEST";
std::vector<int64> vi = {0, 1, 2, 4};
SetContextFeatureInts(feature_key, vi, &sequence);
ASSERT_EQ(GetContextFeatureInts(feature_key, sequence).size(), vi.size());
ASSERT_EQ(GetContextFeatureInts(feature_key, sequence)[3], vi[3]);
ClearContextFeatureInts(feature_key, &sequence);
ASSERT_FALSE(HasFeatureInts(feature_key, sequence));
}
TEST(MediaSequenceTest, RoundTripOpticalFlowEncoded) {
tensorflow::SequenceExample sequence;
std::vector<std::string> flow = {"test", "again"};
@@ -36,6 +36,7 @@ class MediaSequenceTest(tf.test.TestCase):
# context
ms.set_example_id(b"string", example)
ms.set_example_dataset_name(b"string", example)
ms.set_example_dataset_flag_string([b"overal", b"test"], example)
ms.set_clip_media_id(b"string", example)
ms.set_clip_alternative_media_id(b"string", example)
ms.set_clip_encoded_media_bytes(b"string", example)
@@ -76,6 +77,9 @@ class MediaSequenceTest(tf.test.TestCase):
ms.set_instance_segmentation_width(47, example)
ms.set_instance_segmentation_object_class_index((47, 49), example)
ms.set_bbox_parts((b"HEAD", b"TOE"), example)
ms.set_context_feature_floats((47., 35.), example)
ms.set_context_feature_bytes((b"test", b"strings"), example)
ms.set_context_feature_ints((47, 35), example)
# feature lists
ms.add_image_encoded(b"test", example)
ms.add_image_multi_encoded([b"test", b"test"], example)
+2 -1
View File
@@ -17,6 +17,7 @@
#include "absl/strings/str_format.h"
#include "absl/types/variant.h"
#include "mediapipe/framework/formats/location.h"
#include "mediapipe/framework/formats/location_opencv.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/status.h"
@@ -195,7 +196,7 @@ Status TensorsToDetections(const ::tensorflow::Tensor& num_detections,
}
}
LocationData mask_location_data;
mediapipe::Location::CreateCvMaskLocation<float>(mask_image)
mediapipe::CreateCvMaskLocation<float>(mask_image)
.ConvertToProto(&mask_location_data);
location_data->MergeFrom(mask_location_data);
}