From 9b0613a16d113152ac950179e7defbd850f5af97 Mon Sep 17 00:00:00 2001 From: John Douglas <53058687+blittable@users.noreply.github.com> Date: Wed, 23 Oct 2019 05:24:25 +0700 Subject: [PATCH] chore: Add initial benchmarks (#53) --- tonic/Cargo.toml | 8 + tonic/benches/README.md | 37 ++++ tonic/benches/bench_main.rs | 10 ++ .../compiled_protos/diverse_types.rs | 128 ++++++++++++++ .../benchmarks/compiled_protos/helloworld.rs | 166 ++++++++++++++++++ .../benches/benchmarks/compiled_protos/mod.rs | 2 + tonic/benches/benchmarks/mod.rs | 5 + tonic/benches/benchmarks/request_response.rs | 48 +++++ .../request_response_diverse_types.rs | 95 ++++++++++ tonic/benches/benchmarks/utils.rs | 11 ++ .../proto/diverse_types/diverse_types.proto | 79 +++++++++ .../benches/proto/helloworld/helloworld.proto | 37 ++++ 12 files changed, 626 insertions(+) create mode 100755 tonic/benches/README.md create mode 100755 tonic/benches/bench_main.rs create mode 100755 tonic/benches/benchmarks/compiled_protos/diverse_types.rs create mode 100755 tonic/benches/benchmarks/compiled_protos/helloworld.rs create mode 100755 tonic/benches/benchmarks/compiled_protos/mod.rs create mode 100755 tonic/benches/benchmarks/mod.rs create mode 100755 tonic/benches/benchmarks/request_response.rs create mode 100755 tonic/benches/benchmarks/request_response_diverse_types.rs create mode 100755 tonic/benches/benchmarks/utils.rs create mode 100755 tonic/benches/proto/diverse_types/diverse_types.proto create mode 100755 tonic/benches/proto/helloworld/helloworld.proto diff --git a/tonic/Cargo.toml b/tonic/Cargo.toml index 3cfe74d..89c2819 100644 --- a/tonic/Cargo.toml +++ b/tonic/Cargo.toml @@ -37,6 +37,14 @@ openssl = ["openssl1", "tokio-openssl", "tls"] rustls = ["tokio-rustls", "tls"] tls = [] +[[bench]] +name = "bench_main" +harness = false + +[dev-dependencies] +rand = "0.7.2" +criterion = "0.3" + [dependencies] bytes = "0.4" futures-core-preview = "=0.3.0-alpha.19" diff --git a/tonic/benches/README.md b/tonic/benches/README.md new file mode 100755 index 0000000..1d10f07 --- /dev/null +++ b/tonic/benches/README.md @@ -0,0 +1,37 @@ +## Criterion benchmarks for Tonic + +### Running the benchmarks +From the root Tonic directory, `cargo bench` + +After running, the reports can be found in `tonic/target/criterion/report/index.html` + +[Gnuplot](http://www.gnuplot.info/) is required for graph generation. If gnuplot is not installed, Criterion will display: `Gnuplot not found, disabling plotting` at the console. + +### Notes +1) Currently, these benchmarks only test the performance of constructing Tonic Requests and Responses, not over-the-wire throughput. +2) The `thrpt` value generated by Criterion is simply a measure of bytes consumed by the target function. +3) As we are not testing tonic-build compile time, the tests reference pre-compiled .rs files in 'benchmarks/compiled_protos'. +4) The original proto files are in the `proto` directory for reference. +5) This used the Criterion 3.0 `Criterion Group` functionality. Details here: https://docs.rs/criterion/0.3.0/criterion/ + +### Interpreting Results + +Criterion is particularly useful for establishing a first-run baseline and then comparing after code-changes - e.g. `Performance has regressed` below. + +```bash +Request_Response/request/100000 + time: [2.7231 us 2.7588 us 2.7969 us] + thrpt: [33.298 GiB/s 33.758 GiB/s 34.200 GiB/s] + change: + time: [+16.073% +17.871% +19.980%] (p = 0.00 < 0.05) + thrpt: [-16.653% -15.162% -13.847%] + Performance has regressed. +Found 3 outliers among 100 measurements (3.00%) + 1 (1.00%) high mild + 2 (2.00%) high severe +``` + + + + + diff --git a/tonic/benches/bench_main.rs b/tonic/benches/bench_main.rs new file mode 100755 index 0000000..bcd89e4 --- /dev/null +++ b/tonic/benches/bench_main.rs @@ -0,0 +1,10 @@ +use criterion::*; + +mod benchmarks; + +criterion_group!( + benches, + benchmarks::request_response::bench_throughput, + benchmarks::request_response_diverse_types::bench_throughput, +); +criterion_main!(benches); diff --git a/tonic/benches/benchmarks/compiled_protos/diverse_types.rs b/tonic/benches/benchmarks/compiled_protos/diverse_types.rs new file mode 100755 index 0000000..50f471e --- /dev/null +++ b/tonic/benches/benchmarks/compiled_protos/diverse_types.rs @@ -0,0 +1,128 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GoogleMessage1 { + #[prost(string, tag = "1")] + pub field1: std::string::String, + #[prost(string, tag = "9")] + pub field9: std::string::String, + #[prost(string, tag = "18")] + pub field18: std::string::String, + #[prost(bool, tag = "80")] + pub field80: bool, + #[prost(bool, tag = "81")] + pub field81: bool, + #[prost(int32, tag = "2")] + pub field2: i32, + #[prost(int32, tag = "3")] + pub field3: i32, + #[prost(int32, tag = "280")] + pub field280: i32, + #[prost(int32, tag = "6")] + pub field6: i32, + #[prost(int64, tag = "22")] + pub field22: i64, + #[prost(string, tag = "4")] + pub field4: std::string::String, + #[prost(fixed64, repeated, tag = "5")] + pub field5: ::std::vec::Vec, + #[prost(bool, tag = "59")] + pub field59: bool, + #[prost(string, tag = "7")] + pub field7: std::string::String, + #[prost(int32, tag = "16")] + pub field16: i32, + #[prost(int32, tag = "130")] + pub field130: i32, + #[prost(bool, tag = "12")] + pub field12: bool, + #[prost(bool, tag = "17")] + pub field17: bool, + #[prost(bool, tag = "13")] + pub field13: bool, + #[prost(bool, tag = "14")] + pub field14: bool, + #[prost(int32, tag = "104")] + pub field104: i32, + #[prost(int32, tag = "100")] + pub field100: i32, + #[prost(int32, tag = "101")] + pub field101: i32, + #[prost(string, tag = "102")] + pub field102: std::string::String, + #[prost(string, tag = "103")] + pub field103: std::string::String, + #[prost(int32, tag = "29")] + pub field29: i32, + #[prost(bool, tag = "30")] + pub field30: bool, + #[prost(int32, tag = "60")] + pub field60: i32, + #[prost(int32, tag = "271")] + pub field271: i32, + #[prost(int32, tag = "272")] + pub field272: i32, + #[prost(int32, tag = "150")] + pub field150: i32, + #[prost(int32, tag = "23")] + pub field23: i32, + #[prost(bool, tag = "24")] + pub field24: bool, + #[prost(int32, tag = "25")] + pub field25: i32, + #[prost(message, optional, tag = "15")] + pub field15: ::std::option::Option, + #[prost(bool, tag = "78")] + pub field78: bool, + #[prost(int32, tag = "67")] + pub field67: i32, + #[prost(int32, tag = "68")] + pub field68: i32, + #[prost(int32, tag = "128")] + pub field128: i32, + #[prost(string, tag = "129")] + pub field129: std::string::String, + #[prost(int32, tag = "131")] + pub field131: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GoogleMessage1SubMessage { + #[prost(int32, tag = "1")] + pub field1: i32, + #[prost(int32, tag = "2")] + pub field2: i32, + #[prost(int32, tag = "3")] + pub field3: i32, + #[prost(string, tag = "15")] + pub field15: std::string::String, + #[prost(bool, tag = "12")] + pub field12: bool, + #[prost(int64, tag = "13")] + pub field13: i64, + #[prost(int64, tag = "14")] + pub field14: i64, + #[prost(int32, tag = "16")] + pub field16: i32, + #[prost(int32, tag = "19")] + pub field19: i32, + #[prost(bool, tag = "20")] + pub field20: bool, + #[prost(bool, tag = "28")] + pub field28: bool, + #[prost(fixed64, tag = "21")] + pub field21: u64, + #[prost(int32, tag = "22")] + pub field22: i32, + #[prost(bool, tag = "23")] + pub field23: bool, + #[prost(bool, tag = "206")] + pub field206: bool, + #[prost(fixed32, tag = "203")] + pub field203: u32, + #[prost(int32, tag = "204")] + pub field204: i32, + #[prost(string, tag = "205")] + pub field205: std::string::String, + #[prost(uint64, tag = "207")] + pub field207: u64, + #[prost(uint64, tag = "300")] + pub field300: u64, +} diff --git a/tonic/benches/benchmarks/compiled_protos/helloworld.rs b/tonic/benches/benchmarks/compiled_protos/helloworld.rs new file mode 100755 index 0000000..dafe585 --- /dev/null +++ b/tonic/benches/benchmarks/compiled_protos/helloworld.rs @@ -0,0 +1,166 @@ +/// The request message containing the user's name. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HelloRequest { + #[prost(string, tag = "1")] + pub name: std::string::String, +} +/// The response message containing the greetings +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HelloReply { + #[prost(string, tag = "1")] + pub message: std::string::String, +} +#[doc = r" Generated client implementations."] +pub mod client { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = " The greeting service definition."] + pub struct GreeterClient { + inner: tonic::client::Grpc, + } + impl GreeterClient { + #[doc = r" Attempt to create a new client by connecting to a given endpoint."] + pub fn connect(dst: D) -> Result + where + D: std::convert::TryInto, + D::Error: Into, + { + tonic::transport::Endpoint::new(dst).map(|c| Self::new(c.channel())) + } + } + impl GreeterClient + where + T: tonic::client::GrpcService, + T::ResponseBody: Body + HttpBody + Send + 'static, + T::Error: Into, + ::Error: Into + Send, + ::Data: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + #[doc = r" Check if the service is ready."] + pub async fn ready(&mut self) -> Result<(), tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + }) + } + #[doc = " Sends a greeting"] + pub async fn say_hello( + &mut self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.ready().await?; + let codec = tonic::codec::ProstCodec::new(); + let path = http::uri::PathAndQuery::from_static("/helloworld.Greeter/SayHello"); + self.inner.unary(request, path, codec).await + } + } + impl Clone for GreeterClient { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } + } +} +#[doc = r" Generated server implementations."] +pub mod server { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = "Generated trait containing gRPC methods that should be implemented for use with GreeterServer."] + #[async_trait] + pub trait Greeter: Send + Sync + 'static { + #[doc = " Sends a greeting"] + async fn say_hello( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + } + #[doc = " The greeting service definition."] + #[derive(Clone, Debug)] + pub struct GreeterServer { + inner: Arc, + } + #[derive(Clone, Debug)] + #[doc(hidden)] + pub struct GreeterServerSvc { + inner: Arc, + } + impl GreeterServer { + #[doc = "Create a new GreeterServer from a type that implements Greeter."] + pub fn new(inner: T) -> Self { + let inner = Arc::new(inner); + Self::from_shared(inner) + } + pub fn from_shared(inner: Arc) -> Self { + Self { inner } + } + } + impl GreeterServerSvc { + pub fn new(inner: Arc) -> Self { + Self { inner } + } + } + impl Service for GreeterServer { + type Response = GreeterServerSvc; + type Error = Never; + type Future = Ready>; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, _: R) -> Self::Future { + ok(GreeterServerSvc::new(self.inner.clone())) + } + } + impl Service> for GreeterServerSvc { + type Response = http::Response; + type Error = Never; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/helloworld.Greeter/SayHello" => { + struct SayHello(pub Arc); + impl tonic::server::UnaryService for SayHello { + type Response = super::HelloReply; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.say_hello(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let method = SayHello(inner); + let codec = tonic::codec::ProstCodec::new(); + let mut grpc = tonic::server::Grpc::new(codec); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", "12") + .body(tonic::body::BoxBody::empty()) + .unwrap()) + }), + } + } + } +} diff --git a/tonic/benches/benchmarks/compiled_protos/mod.rs b/tonic/benches/benchmarks/compiled_protos/mod.rs new file mode 100755 index 0000000..7ffe6f9 --- /dev/null +++ b/tonic/benches/benchmarks/compiled_protos/mod.rs @@ -0,0 +1,2 @@ +pub mod diverse_types; +pub mod helloworld; diff --git a/tonic/benches/benchmarks/mod.rs b/tonic/benches/benchmarks/mod.rs new file mode 100755 index 0000000..5a8d0ef --- /dev/null +++ b/tonic/benches/benchmarks/mod.rs @@ -0,0 +1,5 @@ +pub mod request_response; +pub mod request_response_diverse_types; + +pub mod compiled_protos; +mod utils; diff --git a/tonic/benches/benchmarks/request_response.rs b/tonic/benches/benchmarks/request_response.rs new file mode 100755 index 0000000..2d15a47 --- /dev/null +++ b/tonic/benches/benchmarks/request_response.rs @@ -0,0 +1,48 @@ +use criterion::*; + +use crate::benchmarks::compiled_protos::helloworld::{HelloReply, HelloRequest}; +use crate::benchmarks::utils; + +fn build_request(_name: String) { + let _request = tonic::Request::new(HelloRequest { name: _name }); +} + +fn build_response(_message: String) { + let _response = tonic::Request::new(HelloReply { message: _message }); +} + +pub fn bench_throughput(c: &mut Criterion) { + let mut group = c.benchmark_group("Request_Response"); + + let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); + + group.plot_config(plot_config); + + let tiny_string = utils::generate_rnd_string(100).unwrap(); + let short_string = utils::generate_rnd_string(1_000).unwrap(); + let medium_string = utils::generate_rnd_string(10_000).unwrap(); + let big_string = utils::generate_rnd_string(100_000).unwrap(); + let huge_string = utils::generate_rnd_string(1_000_000).unwrap(); + let massive_string = utils::generate_rnd_string(10_000_000).unwrap(); + + for size in [ + tiny_string, + short_string, + medium_string, + big_string, + huge_string, + massive_string, + ] + .iter() + { + group.throughput(Throughput::Bytes(size.len() as u64)); + + group.bench_with_input(BenchmarkId::new("request", size.len()), size, |b, i| { + b.iter(|| build_request(i.to_string())) + }); + group.bench_with_input(BenchmarkId::new("response", size.len()), size, |b, i| { + b.iter(|| build_response(i.to_string())) + }); + } + group.finish(); +} diff --git a/tonic/benches/benchmarks/request_response_diverse_types.rs b/tonic/benches/benchmarks/request_response_diverse_types.rs new file mode 100755 index 0000000..16462c7 --- /dev/null +++ b/tonic/benches/benchmarks/request_response_diverse_types.rs @@ -0,0 +1,95 @@ +use criterion::*; + +use crate::benchmarks::compiled_protos::diverse_types::{GoogleMessage1, GoogleMessage1SubMessage}; +use crate::benchmarks::utils; + +fn build_request(_name: String) { + let sub_message = GoogleMessage1SubMessage { + field1: 10, + field2: 20, + field3: 30, + field15: _name, + field12: false, + field13: 70, + field14: 80, + field16: 90, + field19: 100, + field20: true, + field28: false, + field21: 110, + field22: 120, + field23: false, + field206: true, + field203: 233, + field204: 333, + field205: String::from("idiopathic"), + field207: 4000, + field300: 4000, + }; + + let _request = tonic::Request::new(GoogleMessage1 { + field1: String::from("foo"), + field9: String::from("red"), + field18: String::from("red"), + field80: true, + field81: true, + field2: 10, + field3: 30, + field280: 28, + field6: 60, + field22: 220, + field4: String::from("red"), + field5: Vec::new(), + field59: true, + field7: String::from("blue"), + field16: 160, + field130: 13, + field17: false, + field12: true, + field13: true, + field14: false, + field104: 1040, + field100: 50, + field101: 1010, + field102: String::from("green"), + field103: String::from("pink"), + field29: 290, + field30: true, + field60: 601, + field271: 27, + field272: 200, + field150: 15, + field23: 230, + field24: false, + field25: 250, + field15: Some(sub_message), + field78: true, + field67: 670, + field68: 680, + field128: 1280, + field129: String::from("red"), + field131: 300, + }); +} + +pub fn bench_throughput(c: &mut Criterion) { + let mut group = c.benchmark_group("Request_Response_Diverse_Types"); + + //log plot to get everything on the graph + let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); + + group.plot_config(plot_config); + + let tiny_string = utils::generate_rnd_string(100).unwrap(); + let short_string = utils::generate_rnd_string(1_000).unwrap(); + let medium_string = utils::generate_rnd_string(10_000).unwrap(); + + for size in [tiny_string, short_string, medium_string].iter() { + group.throughput(Throughput::Bytes(size.len() as u64)); + + group.bench_with_input(BenchmarkId::new("request", size.len()), size, |b, i| { + b.iter(|| build_request(i.to_string())) + }); + } + group.finish(); +} diff --git a/tonic/benches/benchmarks/utils.rs b/tonic/benches/benchmarks/utils.rs new file mode 100755 index 0000000..5ccc07e --- /dev/null +++ b/tonic/benches/benchmarks/utils.rs @@ -0,0 +1,11 @@ +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; + +pub fn generate_rnd_string(string_size: usize) -> Result> { + let rand_name: String = thread_rng() + .sample_iter(&Alphanumeric) + .take(string_size) + .collect(); + + Ok(rand_name) +} diff --git a/tonic/benches/proto/diverse_types/diverse_types.proto b/tonic/benches/proto/diverse_types/diverse_types.proto new file mode 100755 index 0000000..be5ce00 --- /dev/null +++ b/tonic/benches/proto/diverse_types/diverse_types.proto @@ -0,0 +1,79 @@ +// Benchmark messages for proto3. +// Pinched from the protobuf benchmarks + +syntax = "proto3"; + +package benchmarks.proto3; +option java_package = "com.google.protobuf.benchmarks"; + +// This is the default, but we specify it here explicitly. +option optimize_for = SPEED; + +option cc_enable_arenas = true; + +message GoogleMessage1 { + string field1 = 1; + string field9 = 9; + string field18 = 18; + bool field80 = 80; + bool field81 = 81; + int32 field2 = 2; + int32 field3 = 3; + int32 field280 = 280; + int32 field6 = 6; + int64 field22 = 22; + string field4 = 4; + repeated fixed64 field5 = 5; + bool field59 = 59; + string field7 = 7; + int32 field16 = 16; + int32 field130 = 130; + bool field12 = 12; + bool field17 = 17; + bool field13 = 13; + bool field14 = 14; + int32 field104 = 104; + int32 field100 = 100; + int32 field101 = 101; + string field102 = 102; + string field103 = 103; + int32 field29 = 29; + bool field30 = 30; + int32 field60 = 60; + int32 field271 = 271; + int32 field272 = 272; + int32 field150 = 150; + int32 field23 = 23; + bool field24 = 24; + int32 field25 = 25; + GoogleMessage1SubMessage field15 = 15; + bool field78 = 78; + int32 field67 = 67; + int32 field68 = 68; + int32 field128 = 128; + string field129 = 129; + int32 field131 = 131; +} + +message GoogleMessage1SubMessage { + int32 field1 = 1; + int32 field2 = 2; + int32 field3 = 3; + string field15 = 15; + bool field12 = 12; + int64 field13 = 13; + int64 field14 = 14; + int32 field16 = 16; + int32 field19 = 19; + bool field20 = 20; + bool field28 = 28; + fixed64 field21 = 21; + int32 field22 = 22; + bool field23 = 23; + bool field206 = 206; + fixed32 field203 = 203; + int32 field204 = 204; + string field205 = 205; + uint64 field207 = 207; + uint64 field300 = 300; +} diff --git a/tonic/benches/proto/helloworld/helloworld.proto b/tonic/benches/proto/helloworld/helloworld.proto new file mode 100755 index 0000000..8de5d08 --- /dev/null +++ b/tonic/benches/proto/helloworld/helloworld.proto @@ -0,0 +1,37 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; + +package helloworld; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; +} \ No newline at end of file