chore: Add initial benchmarks (#53)

2019-10-22 18:24:25 -04:00
parent 335a373a40
commit 9b0613a16d
12 changed files with 626 additions and 0 deletions
@@ -37,6 +37,14 @@ openssl = ["openssl1", "tokio-openssl", "tls"]
 rustls = ["tokio-rustls", "tls"]
 tls = []

+[[bench]]
+name = "bench_main"
+harness = false
+
+[dev-dependencies]
+rand = "0.7.2"
+criterion = "0.3"
+
 [dependencies]
 bytes = "0.4"
 futures-core-preview = "=0.3.0-alpha.19"
@@ -0,0 +1,37 @@
+## Criterion benchmarks for Tonic
+
+### Running the benchmarks 
+From the root Tonic directory, `cargo bench`
+
+After running, the reports can be found in `tonic/target/criterion/report/index.html`
+
+[Gnuplot](http://www.gnuplot.info/) is required for graph generation.  If gnuplot is not installed, Criterion will display: `Gnuplot not found, disabling plotting` at the console.
+
+### Notes 
+1) Currently, these benchmarks only test the performance of constructing Tonic Requests and Responses, not over-the-wire throughput. 
+2) The `thrpt` value generated by Criterion is simply a measure of bytes consumed by the target function.
+3) As we are not testing tonic-build compile time, the tests reference pre-compiled .rs files in 'benchmarks/compiled_protos'.
+4) The original proto files are in the `proto` directory for reference.  
+5) This used the Criterion 3.0 `Criterion Group` functionality. Details here: https://docs.rs/criterion/0.3.0/criterion/  
+
+### Interpreting Results
+
+Criterion is particularly useful for establishing a first-run baseline and then comparing after code-changes - e.g. `Performance has regressed` below. 
+
+```bash
+Request_Response/request/100000                                                                             
+                        time:   [2.7231 us 2.7588 us 2.7969 us]
+                        thrpt:  [33.298 GiB/s 33.758 GiB/s 34.200 GiB/s]
+                 change:
+                        time:   [+16.073% +17.871% +19.980%] (p = 0.00 < 0.05)
+                        thrpt:  [-16.653% -15.162% -13.847%]
+                        Performance has regressed.
+Found 3 outliers among 100 measurements (3.00%)
+  1 (1.00%) high mild
+  2 (2.00%) high severe
+```
+
+
+
+
+
@@ -0,0 +1,10 @@
+use criterion::*;
+
+mod benchmarks;
+
+criterion_group!(
+    benches,
+    benchmarks::request_response::bench_throughput,
+    benchmarks::request_response_diverse_types::bench_throughput,
+);
+criterion_main!(benches);
@@ -0,0 +1,128 @@
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GoogleMessage1 {
+    #[prost(string, tag = "1")]
+    pub field1: std::string::String,
+    #[prost(string, tag = "9")]
+    pub field9: std::string::String,
+    #[prost(string, tag = "18")]
+    pub field18: std::string::String,
+    #[prost(bool, tag = "80")]
+    pub field80: bool,
+    #[prost(bool, tag = "81")]
+    pub field81: bool,
+    #[prost(int32, tag = "2")]
+    pub field2: i32,
+    #[prost(int32, tag = "3")]
+    pub field3: i32,
+    #[prost(int32, tag = "280")]
+    pub field280: i32,
+    #[prost(int32, tag = "6")]
+    pub field6: i32,
+    #[prost(int64, tag = "22")]
+    pub field22: i64,
+    #[prost(string, tag = "4")]
+    pub field4: std::string::String,
+    #[prost(fixed64, repeated, tag = "5")]
+    pub field5: ::std::vec::Vec<u64>,
+    #[prost(bool, tag = "59")]
+    pub field59: bool,
+    #[prost(string, tag = "7")]
+    pub field7: std::string::String,
+    #[prost(int32, tag = "16")]
+    pub field16: i32,
+    #[prost(int32, tag = "130")]
+    pub field130: i32,
+    #[prost(bool, tag = "12")]
+    pub field12: bool,
+    #[prost(bool, tag = "17")]
+    pub field17: bool,
+    #[prost(bool, tag = "13")]
+    pub field13: bool,
+    #[prost(bool, tag = "14")]
+    pub field14: bool,
+    #[prost(int32, tag = "104")]
+    pub field104: i32,
+    #[prost(int32, tag = "100")]
+    pub field100: i32,
+    #[prost(int32, tag = "101")]
+    pub field101: i32,
+    #[prost(string, tag = "102")]
+    pub field102: std::string::String,
+    #[prost(string, tag = "103")]
+    pub field103: std::string::String,
+    #[prost(int32, tag = "29")]
+    pub field29: i32,
+    #[prost(bool, tag = "30")]
+    pub field30: bool,
+    #[prost(int32, tag = "60")]
+    pub field60: i32,
+    #[prost(int32, tag = "271")]
+    pub field271: i32,
+    #[prost(int32, tag = "272")]
+    pub field272: i32,
+    #[prost(int32, tag = "150")]
+    pub field150: i32,
+    #[prost(int32, tag = "23")]
+    pub field23: i32,
+    #[prost(bool, tag = "24")]
+    pub field24: bool,
+    #[prost(int32, tag = "25")]
+    pub field25: i32,
+    #[prost(message, optional, tag = "15")]
+    pub field15: ::std::option::Option<GoogleMessage1SubMessage>,
+    #[prost(bool, tag = "78")]
+    pub field78: bool,
+    #[prost(int32, tag = "67")]
+    pub field67: i32,
+    #[prost(int32, tag = "68")]
+    pub field68: i32,
+    #[prost(int32, tag = "128")]
+    pub field128: i32,
+    #[prost(string, tag = "129")]
+    pub field129: std::string::String,
+    #[prost(int32, tag = "131")]
+    pub field131: i32,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GoogleMessage1SubMessage {
+    #[prost(int32, tag = "1")]
+    pub field1: i32,
+    #[prost(int32, tag = "2")]
+    pub field2: i32,
+    #[prost(int32, tag = "3")]
+    pub field3: i32,
+    #[prost(string, tag = "15")]
+    pub field15: std::string::String,
+    #[prost(bool, tag = "12")]
+    pub field12: bool,
+    #[prost(int64, tag = "13")]
+    pub field13: i64,
+    #[prost(int64, tag = "14")]
+    pub field14: i64,
+    #[prost(int32, tag = "16")]
+    pub field16: i32,
+    #[prost(int32, tag = "19")]
+    pub field19: i32,
+    #[prost(bool, tag = "20")]
+    pub field20: bool,
+    #[prost(bool, tag = "28")]
+    pub field28: bool,
+    #[prost(fixed64, tag = "21")]
+    pub field21: u64,
+    #[prost(int32, tag = "22")]
+    pub field22: i32,
+    #[prost(bool, tag = "23")]
+    pub field23: bool,
+    #[prost(bool, tag = "206")]
+    pub field206: bool,
+    #[prost(fixed32, tag = "203")]
+    pub field203: u32,
+    #[prost(int32, tag = "204")]
+    pub field204: i32,
+    #[prost(string, tag = "205")]
+    pub field205: std::string::String,
+    #[prost(uint64, tag = "207")]
+    pub field207: u64,
+    #[prost(uint64, tag = "300")]
+    pub field300: u64,
+}
@@ -0,0 +1,166 @@
+/// The request message containing the user's name.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct HelloRequest {
+    #[prost(string, tag = "1")]
+    pub name: std::string::String,
+}
+/// The response message containing the greetings
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct HelloReply {
+    #[prost(string, tag = "1")]
+    pub message: std::string::String,
+}
+#[doc = r" Generated client implementations."]
+pub mod client {
+    #![allow(unused_variables, dead_code, missing_docs)]
+    use tonic::codegen::*;
+    #[doc = " The greeting service definition."]
+    pub struct GreeterClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl GreeterClient<tonic::transport::Channel> {
+        #[doc = r" Attempt to create a new client by connecting to a given endpoint."]
+        pub fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: std::convert::TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            tonic::transport::Endpoint::new(dst).map(|c| Self::new(c.channel()))
+        }
+    }
+    impl<T> GreeterClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::ResponseBody: Body + HttpBody + Send + 'static,
+        T::Error: Into<StdError>,
+        <T::ResponseBody as HttpBody>::Error: Into<StdError> + Send,
+        <T::ResponseBody as HttpBody>::Data: Into<bytes::Bytes> + Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        #[doc = r" Check if the service is ready."]
+        pub async fn ready(&mut self) -> Result<(), tonic::Status> {
+            self.inner.ready().await.map_err(|e| {
+                tonic::Status::new(
+                    tonic::Code::Unknown,
+                    format!("Service was not ready: {}", e.into()),
+                )
+            })
+        }
+        #[doc = " Sends a greeting"]
+        pub async fn say_hello(
+            &mut self,
+            request: tonic::Request<super::HelloRequest>,
+        ) -> Result<tonic::Response<super::HelloReply>, tonic::Status> {
+            self.ready().await?;
+            let codec = tonic::codec::ProstCodec::new();
+            let path = http::uri::PathAndQuery::from_static("/helloworld.Greeter/SayHello");
+            self.inner.unary(request, path, codec).await
+        }
+    }
+    impl<T: Clone> Clone for GreeterClient<T> {
+        fn clone(&self) -> Self {
+            Self {
+                inner: self.inner.clone(),
+            }
+        }
+    }
+}
+#[doc = r" Generated server implementations."]
+pub mod server {
+    #![allow(unused_variables, dead_code, missing_docs)]
+    use tonic::codegen::*;
+    #[doc = "Generated trait containing gRPC methods that should be implemented for use with GreeterServer."]
+    #[async_trait]
+    pub trait Greeter: Send + Sync + 'static {
+        #[doc = " Sends a greeting"]
+        async fn say_hello(
+            &self,
+            request: tonic::Request<super::HelloRequest>,
+        ) -> Result<tonic::Response<super::HelloReply>, tonic::Status> {
+            Err(tonic::Status::unimplemented("Not yet implemented"))
+        }
+    }
+    #[doc = " The greeting service definition."]
+    #[derive(Clone, Debug)]
+    pub struct GreeterServer<T: Greeter> {
+        inner: Arc<T>,
+    }
+    #[derive(Clone, Debug)]
+    #[doc(hidden)]
+    pub struct GreeterServerSvc<T: Greeter> {
+        inner: Arc<T>,
+    }
+    impl<T: Greeter> GreeterServer<T> {
+        #[doc = "Create a new GreeterServer from a type that implements Greeter."]
+        pub fn new(inner: T) -> Self {
+            let inner = Arc::new(inner);
+            Self::from_shared(inner)
+        }
+        pub fn from_shared(inner: Arc<T>) -> Self {
+            Self { inner }
+        }
+    }
+    impl<T: Greeter> GreeterServerSvc<T> {
+        pub fn new(inner: Arc<T>) -> Self {
+            Self { inner }
+        }
+    }
+    impl<T: Greeter, R> Service<R> for GreeterServer<T> {
+        type Response = GreeterServerSvc<T>;
+        type Error = Never;
+        type Future = Ready<Result<Self::Response, Self::Error>>;
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, _: R) -> Self::Future {
+            ok(GreeterServerSvc::new(self.inner.clone()))
+        }
+    }
+    impl<T: Greeter> Service<http::Request<HyperBody>> for GreeterServerSvc<T> {
+        type Response = http::Response<tonic::body::BoxBody>;
+        type Error = Never;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<HyperBody>) -> Self::Future {
+            let inner = self.inner.clone();
+            match req.uri().path() {
+                "/helloworld.Greeter/SayHello" => {
+                    struct SayHello<T: Greeter>(pub Arc<T>);
+                    impl<T: Greeter> tonic::server::UnaryService<super::HelloRequest> for SayHello<T> {
+                        type Response = super::HelloReply;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::HelloRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move { inner.say_hello(request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = SayHello(inner);
+                        let codec = tonic::codec::ProstCodec::new();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => Box::pin(async move {
+                    Ok(http::Response::builder()
+                        .status(200)
+                        .header("grpc-status", "12")
+                        .body(tonic::body::BoxBody::empty())
+                        .unwrap())
+                }),
+            }
+        }
+    }
+}
@@ -0,0 +1,2 @@
+pub mod diverse_types;
+pub mod helloworld;
@@ -0,0 +1,5 @@
+pub mod request_response;
+pub mod request_response_diverse_types;
+
+pub mod compiled_protos;
+mod utils;
@@ -0,0 +1,48 @@
+use criterion::*;
+
+use crate::benchmarks::compiled_protos::helloworld::{HelloReply, HelloRequest};
+use crate::benchmarks::utils;
+
+fn build_request(_name: String) {
+    let _request = tonic::Request::new(HelloRequest { name: _name });
+}
+
+fn build_response(_message: String) {
+    let _response = tonic::Request::new(HelloReply { message: _message });
+}
+
+pub fn bench_throughput(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Request_Response");
+
+    let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic);
+
+    group.plot_config(plot_config);
+
+    let tiny_string = utils::generate_rnd_string(100).unwrap();
+    let short_string = utils::generate_rnd_string(1_000).unwrap();
+    let medium_string = utils::generate_rnd_string(10_000).unwrap();
+    let big_string = utils::generate_rnd_string(100_000).unwrap();
+    let huge_string = utils::generate_rnd_string(1_000_000).unwrap();
+    let massive_string = utils::generate_rnd_string(10_000_000).unwrap();
+
+    for size in [
+        tiny_string,
+        short_string,
+        medium_string,
+        big_string,
+        huge_string,
+        massive_string,
+    ]
+    .iter()
+    {
+        group.throughput(Throughput::Bytes(size.len() as u64));
+
+        group.bench_with_input(BenchmarkId::new("request", size.len()), size, |b, i| {
+            b.iter(|| build_request(i.to_string()))
+        });
+        group.bench_with_input(BenchmarkId::new("response", size.len()), size, |b, i| {
+            b.iter(|| build_response(i.to_string()))
+        });
+    }
+    group.finish();
+}
@@ -0,0 +1,95 @@
+use criterion::*;
+
+use crate::benchmarks::compiled_protos::diverse_types::{GoogleMessage1, GoogleMessage1SubMessage};
+use crate::benchmarks::utils;
+
+fn build_request(_name: String) {
+    let sub_message = GoogleMessage1SubMessage {
+        field1: 10,
+        field2: 20,
+        field3: 30,
+        field15: _name,
+        field12: false,
+        field13: 70,
+        field14: 80,
+        field16: 90,
+        field19: 100,
+        field20: true,
+        field28: false,
+        field21: 110,
+        field22: 120,
+        field23: false,
+        field206: true,
+        field203: 233,
+        field204: 333,
+        field205: String::from("idiopathic"),
+        field207: 4000,
+        field300: 4000,
+    };
+
+    let _request = tonic::Request::new(GoogleMessage1 {
+        field1: String::from("foo"),
+        field9: String::from("red"),
+        field18: String::from("red"),
+        field80: true,
+        field81: true,
+        field2: 10,
+        field3: 30,
+        field280: 28,
+        field6: 60,
+        field22: 220,
+        field4: String::from("red"),
+        field5: Vec::new(),
+        field59: true,
+        field7: String::from("blue"),
+        field16: 160,
+        field130: 13,
+        field17: false,
+        field12: true,
+        field13: true,
+        field14: false,
+        field104: 1040,
+        field100: 50,
+        field101: 1010,
+        field102: String::from("green"),
+        field103: String::from("pink"),
+        field29: 290,
+        field30: true,
+        field60: 601,
+        field271: 27,
+        field272: 200,
+        field150: 15,
+        field23: 230,
+        field24: false,
+        field25: 250,
+        field15: Some(sub_message),
+        field78: true,
+        field67: 670,
+        field68: 680,
+        field128: 1280,
+        field129: String::from("red"),
+        field131: 300,
+    });
+}
+
+pub fn bench_throughput(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Request_Response_Diverse_Types");
+
+    //log plot to get everything on the graph
+    let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic);
+
+    group.plot_config(plot_config);
+
+    let tiny_string = utils::generate_rnd_string(100).unwrap();
+    let short_string = utils::generate_rnd_string(1_000).unwrap();
+    let medium_string = utils::generate_rnd_string(10_000).unwrap();
+
+    for size in [tiny_string, short_string, medium_string].iter() {
+        group.throughput(Throughput::Bytes(size.len() as u64));
+
+        group.bench_with_input(BenchmarkId::new("request", size.len()), size, |b, i| {
+            b.iter(|| build_request(i.to_string()))
+        });
+    }
+    group.finish();
+}
@@ -0,0 +1,11 @@
+use rand::distributions::Alphanumeric;
+use rand::{thread_rng, Rng};
+
+pub fn generate_rnd_string(string_size: usize) -> Result<String, Box<dyn std::error::Error>> {
+    let rand_name: String = thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(string_size)
+        .collect();
+
+    Ok(rand_name)
+}
@@ -0,0 +1,79 @@
+// Benchmark messages for proto3.
+// Pinched from the protobuf benchmarks
+
+syntax = "proto3";
+
+package benchmarks.proto3;
+option java_package = "com.google.protobuf.benchmarks";
+
+// This is the default, but we specify it here explicitly.
+option optimize_for = SPEED;
+
+option cc_enable_arenas = true;
+
+message GoogleMessage1 {
+  string field1 = 1;
+  string field9 = 9;
+  string field18 = 18;
+  bool field80 = 80;
+  bool field81 = 81;
+  int32 field2 = 2;
+  int32 field3 = 3;
+  int32 field280 = 280;
+  int32 field6 = 6;
+  int64 field22 = 22;
+  string field4 = 4;
+  repeated fixed64 field5 = 5;
+  bool field59 = 59;
+  string field7 = 7;
+  int32 field16 = 16;
+  int32 field130 = 130;
+  bool field12 = 12;
+  bool field17 = 17;
+  bool field13 = 13;
+  bool field14 = 14;
+  int32 field104 = 104;
+  int32 field100 = 100;
+  int32 field101 = 101;
+  string field102 = 102;
+  string field103 = 103;
+  int32 field29 = 29;
+  bool field30 = 30;
+  int32 field60 = 60;
+  int32 field271 = 271;
+  int32 field272 = 272;
+  int32 field150 = 150;
+  int32 field23 = 23;
+  bool field24 = 24;
+  int32 field25 = 25;
+  GoogleMessage1SubMessage field15 = 15;
+  bool field78 = 78;
+  int32 field67 = 67;
+  int32 field68 = 68;
+  int32 field128 = 128;
+  string field129 = 129;
+  int32 field131 = 131;
+}
+
+message GoogleMessage1SubMessage {
+  int32 field1 = 1;
+  int32 field2 = 2;
+  int32 field3 = 3;
+  string field15 = 15;
+  bool field12 = 12;
+  int64 field13 = 13;
+  int64 field14 = 14;
+  int32 field16 = 16;
+  int32 field19 = 19;
+  bool field20  = 20;
+  bool field28 = 28;
+  fixed64 field21 = 21;
+  int32 field22 = 22;
+  bool field23 = 23;
+  bool field206 = 206;
+  fixed32 field203 = 203;
+  int32 field204 = 204;
+  string field205 = 205;
+  uint64 field207 = 207;
+  uint64 field300 = 300;
+}
@@ -0,0 +1,37 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "io.grpc.examples.helloworld";
+option java_outer_classname = "HelloWorldProto";
+
+package helloworld;
+
+// The greeting service definition.
+service Greeter {
+  // Sends a greeting
+  rpc SayHello (HelloRequest) returns (HelloReply) {}
+}
+
+// The request message containing the user's name.
+message HelloRequest {
+  string name = 1;
+}
+
+// The response message containing the greetings
+message HelloReply {
+  string message = 1;
+}