Project import generated by Copybara.

GitOrigin-RevId: 5b23708185311ae39a8605b0c2eff721e7b4939f

mediapipe/calculators/image/BUILD

@@ -630,3 +630,34 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "image_file_properties_calculator",
+    srcs = ["image_file_properties_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_file_properties_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@easyexif",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "image_file_properties_calculator_test",
+    srcs = ["image_file_properties_calculator_test.cc"],
+    data = ["//mediapipe/calculators/image/testdata:test_images"],
+    deps = [
+        ":image_file_properties_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_file_properties_cc_proto",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/calculators/image/image_file_properties_calculator.cc

@@ -0,0 +1,195 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+
+#include "exif.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_file_properties.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+// 35 MM sensor has dimensions 36 mm x 24 mm, so diagonal length is
+// sqrt(36^2 + 24^2).
+static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
+
+::mediapipe::StatusOr<double> ComputeFocalLengthInPixels(
+    int image_width, int image_height, double focal_length_35mm,
+    double focal_length_mm) {
+  // TODO: Allow returning image file properties even when focal length
+  // computation is not possible.
+  if (image_width == 0 || image_height == 0) {
+    return ::mediapipe::InternalError(
+        "Image dimensions should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  if (focal_length_mm == 0) {
+    return ::mediapipe::InternalError(
+        "Focal length in mm should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  if (focal_length_35mm == 0) {
+    return ::mediapipe::InternalError(
+        "Focal length in 35 mm should be non-zero to compute focal length in "
+        "pixels.");
+  }
+  // Derived from
+  // https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length#Calculation.
+  /// Using focal_length_35mm = focal_length_mm * SENSOR_DIAGONAL_35MM /
+  /// sensor_diagonal_mm, we can calculate the diagonal length of the sensor in
+  /// millimeters i.e. sensor_diagonal_mm.
+  double sensor_diagonal_mm =
+      SENSOR_DIAGONAL_35MM / focal_length_35mm * focal_length_mm;
+  // Note that for the following computations, the longer dimension is treated
+  // as image width and the shorter dimension is treated as image height.
+  int width = image_width;
+  int height = image_height;
+  if (image_height > image_width) {
+    width = image_height;
+    height = image_width;
+  }
+  double inv_aspect_ratio = (double)height / width;
+  // Compute sensor width.
+  /// Using Pythagoras theorem, sensor_width^2 + sensor_height^2 =
+  /// sensor_diagonal_mm^2. We can substitute sensor_width / sensor_height with
+  /// the aspect ratio calculated in pixels to compute the sensor width.
+  double sensor_width = std::sqrt((sensor_diagonal_mm * sensor_diagonal_mm) /
+                                  (1.0 + inv_aspect_ratio * inv_aspect_ratio));
+
+  // Compute focal length in pixels.
+  double focal_length_pixels = width * focal_length_mm / sensor_width;
+  return focal_length_pixels;
+}
+
+::mediapipe::StatusOr<ImageFileProperties> GetImageFileProperites(
+    const std::string& image_bytes) {
+  easyexif::EXIFInfo result;
+  int code = result.parseFrom(image_bytes);
+  if (code) {
+    return ::mediapipe::InternalError("Error parsing EXIF, code: " +
+                                      std::to_string(code));
+  }
+
+  ImageFileProperties properties;
+  properties.set_image_width(result.ImageWidth);
+  properties.set_image_height(result.ImageHeight);
+  properties.set_focal_length_mm(result.FocalLength);
+  properties.set_focal_length_35mm(result.FocalLengthIn35mm);
+
+  ASSIGN_OR_RETURN(auto focal_length_pixels,
+                   ComputeFocalLengthInPixels(properties.image_width(),
+                                              properties.image_height(),
+                                              properties.focal_length_35mm(),
+                                              properties.focal_length_mm()));
+  properties.set_focal_length_pixels(focal_length_pixels);
+
+  return properties;
+}
+
+}  // namespace
+
+// Calculator to extract EXIF information from an image file. The input is
+// a std::string containing raw byte data from a file, and the output is an
+// ImageFileProperties proto object with the relevant fields filled in.
+// The calculator accepts the input as a stream or a side packet, and can output
+// the result as a stream or a side packet. The calculator checks that if an
+// output stream is present, it outputs to that stream, and if not, it checks if
+// it can output to a side packet.
+//
+// Example config with input and output streams:
+// node {
+//   calculator: "ImageFilePropertiesCalculator"
+//   input_stream: "image_bytes"
+//   output_stream: "image_properties"
+// }
+// Example config with input and output side packets:
+// node {
+//   calculator: "ImageFilePropertiesCalculator"
+//   input_side_packet: "image_bytes"
+//   output_side_packet: "image_properties"
+// }
+class ImageFilePropertiesCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    if (cc->Inputs().NumEntries() != 0) {
+      RET_CHECK(cc->Inputs().NumEntries() == 1);
+      cc->Inputs().Index(0).Set<std::string>();
+    } else {
+      RET_CHECK(cc->InputSidePackets().NumEntries() == 1);
+      cc->InputSidePackets().Index(0).Set<std::string>();
+    }
+    if (cc->Outputs().NumEntries() != 0) {
+      RET_CHECK(cc->Outputs().NumEntries() == 1);
+      cc->Outputs().Index(0).Set<::mediapipe::ImageFileProperties>();
+    } else {
+      RET_CHECK(cc->OutputSidePackets().NumEntries() == 1);
+      cc->OutputSidePackets().Index(0).Set<::mediapipe::ImageFileProperties>();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    if (cc->InputSidePackets().NumEntries() == 1) {
+      const std::string& image_bytes =
+          cc->InputSidePackets().Index(0).Get<std::string>();
+      ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      read_properties_ = true;
+    }
+
+    if (read_properties_ && cc->OutputSidePackets().NumEntries() == 1) {
+      cc->OutputSidePackets().Index(0).Set(
+          MakePacket<ImageFileProperties>(properties_));
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (cc->Inputs().NumEntries() == 1) {
+      if (cc->Inputs().Index(0).IsEmpty()) {
+        return ::mediapipe::OkStatus();
+      }
+      const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
+      ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      read_properties_ = true;
+    }
+    if (read_properties_) {
+      if (cc->Outputs().NumEntries() == 1) {
+        cc->Outputs().Index(0).AddPacket(
+            MakePacket<ImageFileProperties>(properties_)
+                .At(cc->InputTimestamp()));
+      } else {
+        cc->OutputSidePackets().Index(0).Set(
+            MakePacket<ImageFileProperties>(properties_)
+                .At(::mediapipe::Timestamp::Unset()));
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  ImageFileProperties properties_;
+  bool read_properties_ = false;
+};
+REGISTER_CALCULATOR(ImageFilePropertiesCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/image/image_file_properties_calculator_test.cc

@@ -0,0 +1,134 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <cmath>
+#include <limits>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_file_properties.pb.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kImageFilePath[] =
+    "/mediapipe/calculators/image/testdata/"
+    "front_camera_pixel2.jpg";
+constexpr int kExpectedWidth = 2448;
+constexpr int kExpectedHeight = 3264;
+constexpr double kExpectedFocalLengthMm = 3.38;
+constexpr double kExpectedFocalLengthIn35Mm = 25;
+constexpr double kExpectedFocalLengthPixels = 2357.48;
+
+double RoundToNDecimals(double value, int n) {
+  return std::round(value * pow(10.0, n)) / pow(10.0, n);
+}
+
+TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_stream: "image_bytes"
+        output_stream: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::string>(image_contents).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.Outputs();
+  ASSERT_EQ(1, outputs.NumEntries());
+  const std::vector<Packet>& packets = outputs.Index(0).packets;
+  ASSERT_EQ(1, packets.size());
+  const auto& result = packets[0].Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_side_packet: "image_bytes"
+        output_side_packet: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Index(0) =
+      MakePacket<std::string>(image_contents).At(Timestamp(0));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.OutputSidePackets();
+  EXPECT_EQ(1, outputs.NumEntries());
+  const auto& packet = outputs.Index(0);
+  const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+TEST(ImageFilePropertiesCalculatorTest,
+     ReadsFocalLengthFromJpegStreamToSidePacket) {
+  std::string image_filepath = file::JoinPath("./", kImageFilePath);
+  std::string image_contents;
+  MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
+
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "ImageFilePropertiesCalculator"
+        input_stream: "image_bytes"
+        output_side_packet: "properties"
+      )");
+
+  CalculatorRunner runner(node_config);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::string>(image_contents).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const auto& outputs = runner.OutputSidePackets();
+  EXPECT_EQ(1, outputs.NumEntries());
+  const auto& packet = outputs.Index(0);
+  const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
+  EXPECT_EQ(kExpectedWidth, result.image_width());
+  EXPECT_EQ(kExpectedHeight, result.image_height());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
+  EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
+                   RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
+}
+
+}  // namespace
+}  // namespace mediapipe