Project import generated by Copybara.

GitOrigin-RevId: 5b23708185311ae39a8605b0c2eff721e7b4939f

mediapipe/calculators/core/BUILD

@@ -316,6 +316,37 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "concatenate_normalized_landmark_list_calculator",
+    srcs = ["concatenate_normalized_landmark_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "concatenate_normalized_landmark_list_calculator_test",
+    srcs = ["concatenate_normalized_landmark_list_calculator_test.cc"],
+    deps = [
+        ":concatenate_normalized_landmark_list_calculator",
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_test(
     name = "concatenate_vector_calculator_test",
     srcs = ["concatenate_vector_calculator_test.cc"],

mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc

@@ -0,0 +1,84 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_  // NOLINT
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Concatenates several NormalizedLandmarkList protos following stream index
+// order. This class assumes that every input stream contains a
+// NormalizedLandmarkList proto object.
+class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() != 0);
+    RET_CHECK(cc->Outputs().NumEntries() == 1);
+
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      cc->Inputs().Index(i).Set<NormalizedLandmarkList>();
+    }
+
+    cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    only_emit_if_all_present_ =
+        cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
+            .only_emit_if_all_present();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (only_emit_if_all_present_) {
+      for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+        if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
+      }
+    }
+
+    NormalizedLandmarkList output;
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      if (cc->Inputs().Index(i).IsEmpty()) continue;
+      const NormalizedLandmarkList& input =
+          cc->Inputs().Index(i).Get<NormalizedLandmarkList>();
+      for (int j = 0; j < input.landmark_size(); ++j) {
+        const NormalizedLandmark& input_landmark = input.landmark(j);
+        *output.add_landmark() = input_landmark;
+      }
+    }
+    cc->Outputs().Index(0).AddPacket(
+        MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  bool only_emit_if_all_present_;
+};
+
+REGISTER_CALCULATOR(ConcatenateNormalizedLandmarkListCalculator);
+
+}  // namespace mediapipe
+
+// NOLINTNEXTLINE
+#endif  // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_

mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator_test.cc

@@ -0,0 +1,184 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+constexpr float kLocationValue = 3;
+
+NormalizedLandmarkList GenerateLandmarks(int landmarks_size,
+                                         int value_multiplier) {
+  NormalizedLandmarkList landmarks;
+  for (int i = 0; i < landmarks_size; ++i) {
+    NormalizedLandmark* landmark = landmarks.add_landmark();
+    landmark->set_x(value_multiplier * kLocationValue);
+    landmark->set_y(value_multiplier * kLocationValue);
+    landmark->set_z(value_multiplier * kLocationValue);
+  }
+  return landmarks;
+}
+
+void ValidateCombinedLandmarks(
+    const std::vector<NormalizedLandmarkList>& inputs,
+    const NormalizedLandmarkList& result) {
+  int element_id = 0;
+  int expected_size = 0;
+  for (int i = 0; i < inputs.size(); ++i) {
+    const NormalizedLandmarkList& landmarks_i = inputs[i];
+    expected_size += landmarks_i.landmark_size();
+    for (int j = 0; j < landmarks_i.landmark_size(); ++j) {
+      const NormalizedLandmark& expected = landmarks_i.landmark(j);
+      const NormalizedLandmark& got = result.landmark(element_id);
+      EXPECT_FLOAT_EQ(expected.x(), got.x());
+      EXPECT_FLOAT_EQ(expected.y(), got.y());
+      EXPECT_FLOAT_EQ(expected.z(), got.z());
+      ++element_id;
+    }
+  }
+  EXPECT_EQ(expected_size, result.landmark_size());
+}
+
+void AddInputLandmarkLists(
+    const std::vector<NormalizedLandmarkList>& input_landmarks_vec,
+    int64 timestamp, CalculatorRunner* runner) {
+  for (int i = 0; i < input_landmarks_vec.size(); ++i) {
+    runner->MutableInputs()->Index(i).packets.push_back(
+        MakePacket<NormalizedLandmarkList>(input_landmarks_vec[i])
+            .At(Timestamp(timestamp)));
+  }
+}
+
+TEST(ConcatenateNormalizedLandmarkListCalculatorTest, EmptyVectorInputs) {
+  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  NormalizedLandmarkList empty_list;
+  std::vector<NormalizedLandmarkList> inputs = {empty_list, empty_list,
+                                                empty_list};
+  AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(0, outputs[0].Get<NormalizedLandmarkList>().landmark_size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+}
+
+TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  NormalizedLandmarkList input_0 =
+      GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
+  NormalizedLandmarkList input_1 =
+      GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1);
+  NormalizedLandmarkList input_2 =
+      GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2);
+  std::vector<NormalizedLandmarkList> inputs = {input_0, input_1, input_2};
+  AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const NormalizedLandmarkList& result =
+      outputs[0].Get<NormalizedLandmarkList>();
+  ValidateCombinedLandmarks(inputs, result);
+}
+
+TEST(ConcatenateNormalizedLandmarkListCalculatorTest,
+     TwoInputsAtTwoTimestamps) {
+  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  NormalizedLandmarkList input_0 =
+      GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
+  NormalizedLandmarkList input_1 =
+      GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1);
+  NormalizedLandmarkList input_2 =
+      GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2);
+  std::vector<NormalizedLandmarkList> inputs = {input_0, input_1, input_2};
+  { AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); }
+  { AddInputLandmarkLists(inputs, /*timestamp=*/2, &runner); }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(2, outputs.size());
+  {
+    EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+    const NormalizedLandmarkList& result =
+        outputs[0].Get<NormalizedLandmarkList>();
+    ValidateCombinedLandmarks(inputs, result);
+  }
+  {
+    EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
+    const NormalizedLandmarkList& result =
+        outputs[1].Get<NormalizedLandmarkList>();
+    ValidateCombinedLandmarks(inputs, result);
+  }
+}
+
+TEST(ConcatenateNormalizedLandmarkListCalculatorTest,
+     OneEmptyStreamStillOutput) {
+  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
+                          /*options_string=*/"", /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  NormalizedLandmarkList input_0 =
+      GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
+  std::vector<NormalizedLandmarkList> inputs = {input_0};
+  AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const NormalizedLandmarkList& result =
+      outputs[0].Get<NormalizedLandmarkList>();
+  ValidateCombinedLandmarks(inputs, result);
+}
+
+TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneEmptyStreamNoOutput) {
+  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
+                          /*options_string=*/
+                          "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
+                          "{only_emit_if_all_present: true}",
+                          /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  NormalizedLandmarkList input_0 =
+      GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
+  std::vector<NormalizedLandmarkList> inputs = {input_0};
+  AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(0, outputs.size());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -630,3 +630,34 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "image_file_properties_calculator",
+    srcs = ["image_file_properties_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_file_properties_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@easyexif",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "image_file_properties_calculator_test",
+    srcs = ["image_file_properties_calculator_test.cc"],
+    data = ["//mediapipe/calculators/image/testdata:test_images"],
+    deps = [
+        ":image_file_properties_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_file_properties_cc_proto",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/calculators/image/image_file_properties_calculator.cc

@@ -0,0 +1,195 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+
+#include "exif.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_file_properties.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+// 35 MM sensor has dimensions 36 mm x 24 mm, so diagonal length is
+// sqrt(36^2 + 24^2).
+static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
+
+::mediapipe::StatusOr<double> ComputeFocalLengthInPixels(
+    int image_width, int image_height, double focal_length_35mm,
+    double focal_length_mm) {
+  // TODO: Allow returning image file properties even when focal length
+  // computation is not possible.
+  if (image_width == 0 || image_height == 0) {
+    return ::mediapipe::InternalError(
+        "Image dimensions should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  if (focal_length_mm == 0) {
+    return ::mediapipe::InternalError(
+        "Focal length in mm should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  if (focal_length_35mm == 0) {
+    return ::mediapipe::InternalError(
+        "Focal length in 35 mm should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  // Derived from
+  // https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length#Calculation.
+  /// Using focal_length_35mm = focal_length_mm * SENSOR_DIAGONAL_35MM /
+  /// sensor_diagonal_mm, we can calculate the diagonal length of the sensor in
+  /// millimeters i.e. sensor_diagonal_mm.
+  double sensor_diagonal_mm =
+      SENSOR_DIAGONAL_35MM / focal_length_35mm * focal_length_mm;
+  // Note that for the following computations, the longer dimension is treated
+  // as image width and the shorter dimension is treated as image height.
+  int width = image_width;
+  int height = image_height;
+  if (image_height > image_width) {
+    width = image_height;
+    height = image_width;
+  }
+  double inv_aspect_ratio = (double)height / width;
+  // Compute sensor width.
+  /// Using Pythagoras theorem, sensor_width^2 + sensor_height^2 =
+  /// sensor_diagonal_mm^2. We can substitute sensor_width / sensor_height with
+  /// the aspect ratio calculated in pixels to compute the sensor width.
+  double sensor_width = std::sqrt((sensor_diagonal_mm * sensor_diagonal_mm) /
+                                  (1.0 + inv_aspect_ratio * inv_aspect_ratio));
+
+  // Compute focal length in pixels.
+  double focal_length_pixels = width * focal_length_mm / sensor_width;
+  return focal_length_pixels;
+}
+
+::mediapipe::StatusOr<ImageFileProperties> GetImageFileProperites(
+    const std::string& image_bytes) {
+  easyexif::EXIFInfo result;
+  int code = result.parseFrom(image_bytes);
+  if (code) {
+    return ::mediapipe::InternalError("Error parsing EXIF, code: " +
+                                      std::to_string(code));
+  }
+
+  ImageFileProperties properties;
+  properties.set_image_width(result.ImageWidth);
+  properties.set_image_height(result.ImageHeight);
+  properties.set_focal_length_mm(result.FocalLength);
+  properties.set_focal_length_35mm(result.FocalLengthIn35mm);
+
+  ASSIGN_OR_RETURN(auto focal_length_pixels,
+                   ComputeFocalLengthInPixels(properties.image_width(),
+                                              properties.image_height(),
+                                              properties.focal_length_35mm(),
+                                              properties.focal_length_mm()));
+  properties.set_focal_length_pixels(focal_length_pixels);
+
+  return properties;
+}
+
+}  // namespace
+
+// Calculator to extract EXIF information from an image file. The input is
+// a std::string containing raw byte data from a file, and the output is an
+// ImageFileProperties proto object with the relevant fields filled in.
+// The calculator accepts the input as a stream or a side packet, and can output
+// the result as a stream or a side packet. The calculator checks that if an
+// output stream is present, it outputs to that stream, and if not, it checks if
+// it can output to a side packet.
+//
+// Example config with input and output streams:
+// node {
+//   calculator: "ImageFilePropertiesCalculator"
+//   input_stream: "image_bytes"
+//   output_stream: "image_properties"
+// }
+// Example config with input and output side packets:
+// node {
+//   calculator: "ImageFilePropertiesCalculator"
+//   input_side_packet: "image_bytes"
+//   output_side_packet: "image_properties"
+// }
+class ImageFilePropertiesCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    if (cc->Inputs().NumEntries() != 0) {
+      RET_CHECK(cc->Inputs().NumEntries() == 1);
+      cc->Inputs().Index(0).Set<std::string>();
+    } else {
+      RET_CHECK(cc->InputSidePackets().NumEntries() == 1);
+      cc->InputSidePackets().Index(0).Set<std::string>();
+    }
+    if (cc->Outputs().NumEntries() != 0) {
+      RET_CHECK(cc->Outputs().NumEntries() == 1);
+      cc->Outputs().Index(0).Set<::mediapipe::ImageFileProperties>();
+    } else {
+      RET_CHECK(cc->OutputSidePackets().NumEntries() == 1);
+      cc->OutputSidePackets().Index(0).Set<::mediapipe::ImageFileProperties>();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    if (cc->InputSidePackets().NumEntries() == 1) {
+      const std::string& image_bytes =
+          cc->InputSidePackets().Index(0).Get<std::string>();
+      ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      read_properties_ = true;
+    }
+
+    if (read_properties_ && cc->OutputSidePackets().NumEntries() == 1) {
+      cc->OutputSidePackets().Index(0).Set(
+          MakePacket<ImageFileProperties>(properties_));
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (cc->Inputs().NumEntries() == 1) {
+      if (cc->Inputs().Index(0).IsEmpty()) {
+        return ::mediapipe::OkStatus();
+      }
+      const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
+      ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      read_properties_ = true;
+    }
+    if (read_properties_) {
+      if (cc->Outputs().NumEntries() == 1) {
+        cc->Outputs().Index(0).AddPacket(
+            MakePacket<ImageFileProperties>(properties_)
+                .At(cc->InputTimestamp()));
+      } else {
+        cc->OutputSidePackets().Index(0).Set(
+            MakePacket<ImageFileProperties>(properties_)
+                .At(::mediapipe::Timestamp::Unset()));
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  ImageFileProperties properties_;
+  bool read_properties_ = false;
+};
+REGISTER_CALCULATOR(ImageFilePropertiesCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/image/image_file_properties_calculator_test.cc

@@ -0,0 +1,134 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <cmath>
+#include <limits>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_file_properties.pb.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kImageFilePath[] =
+    "/mediapipe/calculators/image/testdata/"
+    "front_camera_pixel2.jpg";
+constexpr int kExpectedWidth = 2448;
+constexpr int kExpectedHeight = 3264;
+constexpr double kExpectedFocalLengthMm = 3.38;
+constexpr double kExpectedFocalLengthIn35Mm = 25;
+constexpr double kExpectedFocalLengthPixels = 2357.48;
+
+double RoundToNDecimals(double value, int n) {
+  return std::round(value * pow(10.0, n)) / pow(10.0, n);
+}
+
+TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_stream: "image_bytes"
+        output_stream: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::string>(image_contents).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.Outputs();
+  ASSERT_EQ(1, outputs.NumEntries());
+  const std::vector<Packet>& packets = outputs.Index(0).packets;
+  ASSERT_EQ(1, packets.size());
+  const auto& result = packets[0].Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_side_packet: "image_bytes"
+        output_side_packet: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Index(0) =
+      MakePacket<std::string>(image_contents).At(Timestamp(0));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.OutputSidePackets();
+  EXPECT_EQ(1, outputs.NumEntries());
+  const auto& packet = outputs.Index(0);
+  const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+TEST(ImageFilePropertiesCalculatorTest,
+     ReadsFocalLengthFromJpegStreamToSidePacket) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_stream: "image_bytes"
+        output_side_packet: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::string>(image_contents).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.OutputSidePackets();
+  EXPECT_EQ(1, outputs.NumEntries());
+  const auto& packet = outputs.Index(0);
+  const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -160,8 +160,8 @@ class AnnotationOverlayCalculator : public CalculatorBase {
   GLuint image_mat_tex_ = 0;  // Overlay drawing image for GPU.
   int width_ = 0;
   int height_ = 0;
-  int width_gpu_ = 0;  // Size of overlay drawing texture.
-  int height_gpu_ = 0;
+  int width_canvas_ = 0;  // Size of overlay drawing texture canvas.
+  int height_canvas_ = 0;
 #endif  //  MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(AnnotationOverlayCalculator);
@@ -250,6 +250,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   // Initialize the helper renderer library.
   renderer_ = absl::make_unique<AnnotationRenderer>();
   renderer_->SetFlipTextVertically(options_.flip_text_vertically());
+  if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor());
 
   // Set the output header based on the input header (if present).
   const char* input_tag = use_gpu_ ? kInputFrameTagGpu : kInputFrameTag;
@@ -391,8 +392,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
     glBindTexture(GL_TEXTURE_2D, image_mat_tex_);
-    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width_gpu_, height_gpu_, GL_RGB,
-                    GL_UNSIGNED_BYTE, overlay_image);
+    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width_canvas_, height_canvas_,
+                    GL_RGB, GL_UNSIGNED_BYTE, overlay_image);
     glBindTexture(GL_TEXTURE_2D, 0);
   }
 
@@ -494,12 +495,13 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     if (format != mediapipe::ImageFormat::SRGBA &&
         format != mediapipe::ImageFormat::SRGB)
       RET_CHECK_FAIL() << "Unsupported GPU input format: " << format;
-    image_mat = absl::make_unique<cv::Mat>(height_gpu_, width_gpu_, CV_8UC3);
+    image_mat =
+        absl::make_unique<cv::Mat>(height_canvas_, width_canvas_, CV_8UC3);
     memset(image_mat->data, kAnnotationBackgroundColor,
-           height_gpu_ * width_gpu_ * image_mat->elemSize());
+           height_canvas_ * width_canvas_ * image_mat->elemSize());
   } else {
     image_mat = absl::make_unique<cv::Mat>(
-        height_gpu_, width_gpu_, CV_8UC3,
+        height_canvas_, width_canvas_, CV_8UC3,
         cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(),
                    options_.canvas_color().b()));
   }
@@ -646,8 +648,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     width_ = RoundUp(options_.canvas_width_px(), alignment);
     height_ = RoundUp(options_.canvas_height_px(), alignment);
   }
-  width_gpu_ = RoundUp(width_ * scale_factor, alignment);
-  height_gpu_ = RoundUp(height_ * scale_factor, alignment);
+  width_canvas_ = RoundUp(width_ * scale_factor, alignment);
+  height_canvas_ = RoundUp(height_ * scale_factor, alignment);
 
   // Init texture for opencv rendered frame.
   {
@@ -655,8 +657,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     glBindTexture(GL_TEXTURE_2D, image_mat_tex_);
     // TODO
     // OpenCV only renders to RGB images, not RGBA. Ideally this should be RGBA.
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, width_gpu_, height_gpu_, 0, GL_RGB,
-                 GL_UNSIGNED_BYTE, nullptr);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, width_canvas_, height_canvas_, 0,
+                 GL_RGB, GL_UNSIGNED_BYTE, nullptr);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);

mediapipe/calculators/util/annotation_overlay_calculator.proto

@@ -50,7 +50,5 @@ message AnnotationOverlayCalculatorOptions {
   // This can be used to speed up annotation by drawing the annotation on an
   // intermediate image with a reduced scale, e.g. 0.5 (of the input image width
   // and height), before resizing and overlaying it on top of the input image.
-  // Should only be used if *all* render data uses normalized coordinates
-  // (or absolute coordinates are updated to scale accordingly).
   optional float gpu_scale_factor = 7 [default = 1.0];
 }

mediapipe/calculators/video/BUILD

@@ -316,6 +316,7 @@ cc_library(
         "//mediapipe/util/tracking",
         "//mediapipe/util/tracking:box_tracker",
         "//mediapipe/util/tracking:tracking_visualization_utilities",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/container:node_hash_set",
         "@com_google_absl//absl/strings",
     ],

mediapipe/calculators/video/box_tracker_calculator.cc

@@ -18,6 +18,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/container/node_hash_set.h"
 #include "absl/strings/numbers.h"
 #include "mediapipe/calculators/video/box_tracker_calculator.pb.h"
@@ -238,6 +239,11 @@ class BoxTrackerCalculator : public CalculatorBase {
   // Queued track time requests.
   std::vector<Timestamp> queued_track_requests_;
 
+  // Stores the tracked ids that have been discarded actively, from continuous
+  // tracking data. It may accumulate across multiple frames. Once consumed, it
+  // should be cleared immediately.
+  absl::flat_hash_set<int> actively_discarded_tracked_ids_;
+
   // Add smooth transition between re-acquisition and previous tracked boxes.
   // `result_box` is the tracking result of one specific timestamp. The smoothed
   // result will be updated in place.
@@ -1144,9 +1150,16 @@ void BoxTrackerCalculator::StreamTrack(const TrackingData& data,
   CHECK(box_map);
   CHECK(failed_ids);
 
+  // Cache the actively discarded tracked ids from the new tracking data.
+  for (const int discarded_id :
+       data.motion_data().actively_discarded_tracked_ids()) {
+    actively_discarded_tracked_ids_.insert(discarded_id);
+  }
+
   // Track all existing boxes by one frame.
   MotionVectorFrame mvf;  // Holds motion from current to previous frame.
   MotionVectorFrameFromTrackingData(data, &mvf);
+  mvf.actively_discarded_tracked_ids = &actively_discarded_tracked_ids_;
 
   if (forward) {
     MotionVectorFrame mvf_inverted;