feat(build): Better support for custom codecs (#999)

BREAKING CHANGE: `CODEC_PATH` moved from const to fn
This commit is contained in:
Brandon Williams
2022-05-05 09:52:03 -07:00
committed by GitHub
parent 1b0b525115
commit de2e4ac077
10 changed files with 694 additions and 21 deletions
+8
View File
@@ -186,6 +186,14 @@ path = "src/streaming/client.rs"
name = "streaming-server"
path = "src/streaming/server.rs"
[[bin]]
name = "json-codec-client"
path = "src/json-codec/client.rs"
[[bin]]
name = "json-codec-server"
path = "src/json-codec/server.rs"
[dependencies]
async-stream = "0.3"
futures = { version = "0.3", default-features = false, features = ["alloc"] }
+27 -2
View File
@@ -1,5 +1,4 @@
use std::env;
use std::path::PathBuf;
use std::{env, path::PathBuf};
fn main() {
tonic_build::configure()
@@ -30,4 +29,30 @@ fn main() {
&["proto/googleapis"],
)
.unwrap();
build_json_codec_service();
}
// Manually define the json.helloworld.Greeter service which used a custom JsonCodec to use json
// serialization instead of protobuf for sending messages on the wire.
// This will result in generated client and server code which relies on its request, response and
// codec types being defined in a module `crate::common`.
//
// See the client/server examples defined in `src/json-codec` for more information.
fn build_json_codec_service() {
let greeter_service = tonic_build::manual::Service::builder()
.name("Greeter")
.package("json.helloworld")
.method(
tonic_build::manual::Method::builder()
.name("say_hello")
.route_name("SayHello")
.input_type("crate::common::HelloRequest")
.output_type("crate::common::HelloResponse")
.codec_path("crate::common::JsonCodec")
.build(),
)
.build();
tonic_build::manual::Builder::new().compile(&[greeter_service]);
}
+28
View File
@@ -0,0 +1,28 @@
//! A HelloWorld example that uses JSON instead of protobuf as the message serialization format.
//!
//! Generated code is the output of codegen as defined in the `build_json_codec_service` function
//! in the `examples/build.rs` file. As defined there, the generated code assumes that a module
//! `crate::common` exists which defines `HelloRequest`, `HelloResponse`, and `JsonCodec`.
pub mod common;
use common::HelloRequest;
pub mod hello_world {
include!(concat!(env!("OUT_DIR"), "/json.helloworld.Greeter.rs"));
}
use hello_world::greeter_client::GreeterClient;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut client = GreeterClient::connect("http://[::1]:50051").await?;
let request = tonic::Request::new(HelloRequest {
name: "Tonic".into(),
});
let response = client.say_hello(request).await?;
println!("RESPONSE={:?}", response);
Ok(())
}
+80
View File
@@ -0,0 +1,80 @@
//! This module defines common request/response types as well as the JsonCodec that is used by the
//! json.helloworld.Greeter service which is defined manually (instead of via proto files) by the
//! `build_json_codec_service` function in the `examples/build.rs` file.
use bytes::{Buf, BufMut};
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;
use tonic::{
codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder},
Status,
};
#[derive(Debug, Deserialize, Serialize)]
pub struct HelloRequest {
pub name: String,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct HelloResponse {
pub message: String,
}
#[derive(Debug)]
pub struct JsonEncoder<T>(PhantomData<T>);
impl<T: serde::Serialize> Encoder for JsonEncoder<T> {
type Item = T;
type Error = Status;
fn encode(&mut self, item: Self::Item, buf: &mut EncodeBuf<'_>) -> Result<(), Self::Error> {
serde_json::to_writer(buf.writer(), &item).map_err(|e| Status::internal(e.to_string()))
}
}
#[derive(Debug)]
pub struct JsonDecoder<U>(PhantomData<U>);
impl<U: serde::de::DeserializeOwned> Decoder for JsonDecoder<U> {
type Item = U;
type Error = Status;
fn decode(&mut self, buf: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> {
if !buf.has_remaining() {
return Ok(None);
}
let item: Self::Item =
serde_json::from_reader(buf.reader()).map_err(|e| Status::internal(e.to_string()))?;
Ok(Some(item))
}
}
/// A [`Codec`] that implements `application/grpc+json` via the serde library.
#[derive(Debug, Clone)]
pub struct JsonCodec<T, U>(PhantomData<(T, U)>);
impl<T, U> Default for JsonCodec<T, U> {
fn default() -> Self {
Self(PhantomData)
}
}
impl<T, U> Codec for JsonCodec<T, U>
where
T: serde::Serialize + Send + 'static,
U: serde::de::DeserializeOwned + Send + 'static,
{
type Encode = T;
type Decode = U;
type Encoder = JsonEncoder<T>;
type Decoder = JsonDecoder<U>;
fn encoder(&mut self) -> Self::Encoder {
JsonEncoder(PhantomData)
}
fn decoder(&mut self) -> Self::Decoder {
JsonDecoder(PhantomData)
}
}
+48
View File
@@ -0,0 +1,48 @@
//! A HelloWorld example that uses JSON instead of protobuf as the message serialization format.
//!
//! Generated code is the output of codegen as defined in the `build_json_codec_service` function
//! in the `examples/build.rs` file. As defined there, the generated code assumes that a module
//! `crate::common` exists which defines `HelloRequest`, `HelloResponse`, and `JsonCodec`.
use tonic::{transport::Server, Request, Response, Status};
pub mod common;
use common::{HelloRequest, HelloResponse};
pub mod hello_world {
include!(concat!(env!("OUT_DIR"), "/json.helloworld.Greeter.rs"));
}
use hello_world::greeter_server::{Greeter, GreeterServer};
#[derive(Default)]
pub struct MyGreeter {}
#[tonic::async_trait]
impl Greeter for MyGreeter {
async fn say_hello(
&self,
request: Request<HelloRequest>,
) -> Result<Response<HelloResponse>, Status> {
println!("Got a request from {:?}", request.remote_addr());
let reply = HelloResponse {
message: format!("Hello {}!", request.into_inner().name),
};
Ok(Response::new(reply))
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let addr = "[::1]:50051".parse().unwrap();
let greeter = MyGreeter::default();
println!("GreeterServer listening on {}", addr);
Server::builder()
.add_service(GreeterServer::new(greeter))
.serve(addr)
.await?;
Ok(())
}
+4 -4
View File
@@ -167,7 +167,7 @@ fn generate_unary<T: Method>(
compile_well_known_types: bool,
path: String,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let ident = format_ident!("{}", method.name());
let (request, response) = method.request_response_name(proto_path, compile_well_known_types);
@@ -192,7 +192,7 @@ fn generate_server_streaming<T: Method>(
compile_well_known_types: bool,
path: String,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let ident = format_ident!("{}", method.name());
let (request, response) = method.request_response_name(proto_path, compile_well_known_types);
@@ -218,7 +218,7 @@ fn generate_client_streaming<T: Method>(
compile_well_known_types: bool,
path: String,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let ident = format_ident!("{}", method.name());
let (request, response) = method.request_response_name(proto_path, compile_well_known_types);
@@ -244,7 +244,7 @@ fn generate_streaming<T: Method>(
compile_well_known_types: bool,
path: String,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let ident = format_ident!("{}", method.name());
let (request, response) = method.request_response_name(proto_path, compile_well_known_types);
+4 -5
View File
@@ -79,6 +79,8 @@ mod prost;
#[cfg_attr(docsrs, doc(cfg(feature = "prost")))]
pub use prost::{compile_protos, configure, Builder};
pub mod manual;
/// Service code generation for client
pub mod client;
/// Service code generation for Server
@@ -91,9 +93,6 @@ pub mod server;
/// to allow any codegen module to generate service
/// abstractions.
pub trait Service {
/// Path to the codec.
const CODEC_PATH: &'static str;
/// Comment type.
type Comment: AsRef<str>;
@@ -119,8 +118,6 @@ pub trait Service {
/// to generate abstraction implementations for
/// the provided methods.
pub trait Method {
/// Path to the codec.
const CODEC_PATH: &'static str;
/// Comment type.
type Comment: AsRef<str>;
@@ -128,6 +125,8 @@ pub trait Method {
fn name(&self) -> &str;
/// Identifier used to generate type name.
fn identifier(&self) -> &str;
/// Path to the codec.
fn codec_path(&self) -> &str;
/// Method is streamed by client.
fn client_streaming(&self) -> bool;
/// Method is streamed by server.
+482
View File
@@ -0,0 +1,482 @@
//! This module provides utilities for generating `tonic` service stubs and clients
//! purely in Rust without the need of `proto` files. It also enables you to set a custom `Codec`
//! if you want to use a custom serialization format other than `protobuf`.
//!
//! # Example
//!
//! ```rust,no_run
//! fn main() -> Result<(), Box<dyn std::error::Error>> {
//! let greeter_service = tonic_build::manual::Service::builder()
//! .name("Greeter")
//! .package("helloworld")
//! .method(
//! tonic_build::manual::Method::builder()
//! .name("say_hello")
//! .route_name("SayHello")
//! // Provide the path to the Request type
//! .input_type("crate::HelloRequest")
//! // Provide the path to the Response type
//! .output_type("super::HelloResponse")
//! // Provide the path to the Codec to use
//! .codec_path("crate::JsonCodec")
//! .build(),
//! )
//! .build();
//!
//! tonic_build::manual::Builder::new().compile(&[greeter_service]);
//! Ok(())
//! }
//! ```
use super::{client, server, Attributes};
use proc_macro2::TokenStream;
use quote::ToTokens;
use std::{
fs,
path::{Path, PathBuf},
};
/// Service builder.
///
/// This builder can be used to manually define a gRPC service in rust code without the use of a
/// .proto file.
///
/// # Example
///
/// ```
/// # use tonic_build::manual::Service;
/// let greeter_service = Service::builder()
/// .name("Greeter")
/// .package("helloworld")
/// // Add various methods to the service
/// // .method()
/// .build();
/// ```
#[derive(Debug, Default)]
pub struct ServiceBuilder {
/// The service name in Rust style.
name: Option<String>,
/// The package name as it appears in the .proto file.
package: Option<String>,
/// The service comments.
comments: Vec<String>,
/// The service methods.
methods: Vec<Method>,
}
impl ServiceBuilder {
/// Set the name for this Service.
///
/// This value will be used both as the base for the generated rust types and service trait as
/// well as part of the route for calling this service. Routes have the form:
/// `/<package_name>.<service_name>/<method_route_name>`
pub fn name(mut self, name: impl AsRef<str>) -> Self {
self.name = Some(name.as_ref().to_owned());
self
}
/// Set the package this Service is part of.
///
/// This value will be used as part of the route for calling this service.
/// Routes have the form: `/<package_name>.<service_name>/<method_route_name>`
pub fn package(mut self, package: impl AsRef<str>) -> Self {
self.package = Some(package.as_ref().to_owned());
self
}
/// Add a comment string that should be included as a doc comment for this Service.
pub fn comment(mut self, comment: impl AsRef<str>) -> Self {
self.comments.push(comment.as_ref().to_owned());
self
}
/// Adds a Method to this Service.
pub fn method(mut self, method: Method) -> Self {
self.methods.push(method);
self
}
/// Build a Service.
///
/// Panics if `name` or `package` weren't set.
pub fn build(self) -> Service {
Service {
name: self.name.unwrap(),
comments: self.comments,
package: self.package.unwrap(),
methods: self.methods,
}
}
}
/// A service descriptor.
#[derive(Debug)]
pub struct Service {
/// The service name in Rust style.
name: String,
/// The package name as it appears in the .proto file.
package: String,
/// The service comments.
comments: Vec<String>,
/// The service methods.
methods: Vec<Method>,
}
impl Service {
/// Create a new `ServiceBuilder`
pub fn builder() -> ServiceBuilder {
ServiceBuilder::default()
}
}
impl crate::Service for Service {
type Comment = String;
type Method = Method;
fn name(&self) -> &str {
&self.name
}
fn package(&self) -> &str {
&self.package
}
fn identifier(&self) -> &str {
&self.name
}
fn methods(&self) -> &[Self::Method] {
&self.methods
}
fn comment(&self) -> &[Self::Comment] {
&self.comments
}
}
/// A service method descriptor.
#[derive(Debug)]
pub struct Method {
/// The name of the method in Rust style.
name: String,
/// The name of the method as should be used when constructing a route
route_name: String,
/// The method comments.
comments: Vec<String>,
/// The input Rust type.
input_type: String,
/// The output Rust type.
output_type: String,
/// Identifies if client streams multiple client messages.
client_streaming: bool,
/// Identifies if server streams multiple server messages.
server_streaming: bool,
/// The path to the codec to use for this method
codec_path: String,
}
impl Method {
/// Create a new `MethodBuilder`
pub fn builder() -> MethodBuilder {
MethodBuilder::default()
}
}
impl crate::Method for Method {
type Comment = String;
fn name(&self) -> &str {
&self.name
}
fn identifier(&self) -> &str {
&self.route_name
}
fn codec_path(&self) -> &str {
&self.codec_path
}
fn client_streaming(&self) -> bool {
self.client_streaming
}
fn server_streaming(&self) -> bool {
self.server_streaming
}
fn comment(&self) -> &[Self::Comment] {
&self.comments
}
fn request_response_name(
&self,
_proto_path: &str,
_compile_well_known_types: bool,
) -> (TokenStream, TokenStream) {
let request = syn::parse_str::<syn::Path>(&self.input_type)
.unwrap()
.to_token_stream();
let response = syn::parse_str::<syn::Path>(&self.output_type)
.unwrap()
.to_token_stream();
(request, response)
}
}
/// Method builder.
///
/// This builder can be used to manually define gRPC method, which can be added to a gRPC service,
/// in rust code without the use of a .proto file.
///
/// # Example
///
/// ```
/// # use tonic_build::manual::Method;
/// let say_hello_method = Method::builder()
/// .name("say_hello")
/// .route_name("SayHello")
/// // Provide the path to the Request type
/// .input_type("crate::common::HelloRequest")
/// // Provide the path to the Response type
/// .output_type("crate::common::HelloResponse")
/// // Provide the path to the Codec to use
/// .codec_path("crate::common::JsonCodec")
/// .build();
/// ```
#[derive(Debug, Default)]
pub struct MethodBuilder {
/// The name of the method in Rust style.
name: Option<String>,
/// The name of the method as should be used when constructing a route
route_name: Option<String>,
/// The method comments.
comments: Vec<String>,
/// The input Rust type.
input_type: Option<String>,
/// The output Rust type.
output_type: Option<String>,
/// Identifies if client streams multiple client messages.
client_streaming: bool,
/// Identifies if server streams multiple server messages.
server_streaming: bool,
/// The path to the codec to use for this method
codec_path: Option<String>,
}
impl MethodBuilder {
/// Set the name for this Method.
///
/// This value will be used for generating the client functions for calling this Method.
///
/// Generally this is formatted in snake_case.
pub fn name(mut self, name: impl AsRef<str>) -> Self {
self.name = Some(name.as_ref().to_owned());
self
}
/// Set the route_name for this Method.
///
/// This value will be used as part of the route for calling this method.
/// Routes have the form: `/<package_name>.<service_name>/<method_route_name>`
///
/// Generally this is formatted in PascalCase.
pub fn route_name(mut self, route_name: impl AsRef<str>) -> Self {
self.route_name = Some(route_name.as_ref().to_owned());
self
}
/// Add a comment string that should be included as a doc comment for this Method.
pub fn comment(mut self, comment: impl AsRef<str>) -> Self {
self.comments.push(comment.as_ref().to_owned());
self
}
/// Set the path to the Rust type that should be use for the Request type of this method.
pub fn input_type(mut self, input_type: impl AsRef<str>) -> Self {
self.input_type = Some(input_type.as_ref().to_owned());
self
}
/// Set the path to the Rust type that should be use for the Response type of this method.
pub fn output_type(mut self, output_type: impl AsRef<str>) -> Self {
self.output_type = Some(output_type.as_ref().to_owned());
self
}
/// Set the path to the Rust type that should be used as the `Codec` for this method.
///
/// Currently the codegen assumes that this type implements `Default`.
pub fn codec_path(mut self, codec_path: impl AsRef<str>) -> Self {
self.codec_path = Some(codec_path.as_ref().to_owned());
self
}
/// Sets if the Method request from the client is streamed.
pub fn client_streaming(mut self) -> Self {
self.client_streaming = true;
self
}
/// Sets if the Method response from the server is streamed.
pub fn server_streaming(mut self) -> Self {
self.server_streaming = true;
self
}
/// Build a Method
///
/// Panics if `name`, `route_name`, `input_type`, `output_type`, or `codec_path` weren't set.
pub fn build(self) -> Method {
Method {
name: self.name.unwrap(),
route_name: self.route_name.unwrap(),
comments: self.comments,
input_type: self.input_type.unwrap(),
output_type: self.output_type.unwrap(),
client_streaming: self.client_streaming,
server_streaming: self.server_streaming,
codec_path: self.codec_path.unwrap(),
}
}
}
struct ServiceGenerator {
builder: Builder,
clients: TokenStream,
servers: TokenStream,
}
impl ServiceGenerator {
fn generate(&mut self, service: &Service) {
if self.builder.build_server {
let server = server::generate(
service,
true, // emit_package,
"", // proto_path, -- not used
false, // compile_well_known_types -- not used
&Attributes::default(),
);
self.servers.extend(server);
}
if self.builder.build_client {
let client = client::generate(
service,
true, // emit_package,
"", // proto_path, -- not used
false, // compile_well_known_types, -- not used
&Attributes::default(),
);
self.clients.extend(client);
}
}
fn finalize(&mut self, buf: &mut String) {
if self.builder.build_client && !self.clients.is_empty() {
let clients = &self.clients;
let client_service = quote::quote! {
#clients
};
let ast: syn::File = syn::parse2(client_service).expect("not a valid tokenstream");
let code = prettyplease::unparse(&ast);
buf.push_str(&code);
self.clients = TokenStream::default();
}
if self.builder.build_server && !self.servers.is_empty() {
let servers = &self.servers;
let server_service = quote::quote! {
#servers
};
let ast: syn::File = syn::parse2(server_service).expect("not a valid tokenstream");
let code = prettyplease::unparse(&ast);
buf.push_str(&code);
self.servers = TokenStream::default();
}
}
}
/// Service generator builder.
#[derive(Debug)]
pub struct Builder {
build_server: bool,
build_client: bool,
out_dir: Option<PathBuf>,
}
impl Default for Builder {
fn default() -> Self {
Self {
build_server: true,
build_client: true,
out_dir: None,
}
}
}
impl Builder {
/// Create a new Builder
pub fn new() -> Self {
Self::default()
}
/// Enable or disable gRPC client code generation.
///
/// Defaults to enabling client code generation.
pub fn build_client(mut self, enable: bool) -> Self {
self.build_client = enable;
self
}
/// Enable or disable gRPC server code generation.
///
/// Defaults to enabling server code generation.
pub fn build_server(mut self, enable: bool) -> Self {
self.build_server = enable;
self
}
/// Set the output directory to generate code to.
///
/// Defaults to the `OUT_DIR` environment variable.
pub fn out_dir(mut self, out_dir: impl AsRef<Path>) -> Self {
self.out_dir = Some(out_dir.as_ref().to_path_buf());
self
}
/// Performs code generation for the provided services.
///
/// Generated services will be output into the directory specified by `out_dir`
/// with files named `<package_name>.<service_name>.rs`.
pub fn compile(self, services: &[Service]) {
let out_dir = if let Some(out_dir) = self.out_dir.as_ref() {
out_dir.clone()
} else {
PathBuf::from(std::env::var("OUT_DIR").unwrap())
};
let mut generator = ServiceGenerator {
builder: self,
clients: TokenStream::default(),
servers: TokenStream::default(),
};
for service in services {
generator.generate(service);
let mut output = String::new();
generator.finalize(&mut output);
let out_file = out_dir.join(format!("{}.{}.rs", service.package, service.name));
fs::write(out_file, output).unwrap();
}
}
}
+9 -6
View File
@@ -2,9 +2,11 @@ use super::{client, server, Attributes};
use proc_macro2::TokenStream;
use prost_build::{Config, Method, Service};
use quote::ToTokens;
use std::ffi::OsString;
use std::io;
use std::path::{Path, PathBuf};
use std::{
ffi::OsString,
io,
path::{Path, PathBuf},
};
/// Configure `tonic-build` code generation.
///
@@ -51,8 +53,6 @@ const PROST_CODEC_PATH: &str = "tonic::codec::ProstCodec";
const NON_PATH_TYPE_ALLOWLIST: &[&str] = &["()"];
impl crate::Service for Service {
const CODEC_PATH: &'static str = PROST_CODEC_PATH;
type Method = Method;
type Comment = String;
@@ -78,7 +78,6 @@ impl crate::Service for Service {
}
impl crate::Method for Method {
const CODEC_PATH: &'static str = PROST_CODEC_PATH;
type Comment = String;
fn name(&self) -> &str {
@@ -89,6 +88,10 @@ impl crate::Method for Method {
&self.proto_name
}
fn codec_path(&self) -> &str {
PROST_CODEC_PATH
}
fn client_streaming(&self) -> bool {
self.client_streaming
}
+4 -4
View File
@@ -366,7 +366,7 @@ fn generate_unary<T: Method>(
method_ident: Ident,
server_trait: Ident,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let service_ident = quote::format_ident!("{}Svc", method.identifier());
@@ -415,7 +415,7 @@ fn generate_server_streaming<T: Method>(
method_ident: Ident,
server_trait: Ident,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let service_ident = quote::format_ident!("{}Svc", method.identifier());
@@ -470,7 +470,7 @@ fn generate_client_streaming<T: Method>(
let service_ident = quote::format_ident!("{}Svc", method.identifier());
let (request, response) = method.request_response_name(proto_path, compile_well_known_types);
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
quote! {
#[allow(non_camel_case_types)]
@@ -517,7 +517,7 @@ fn generate_streaming<T: Method>(
method_ident: Ident,
server_trait: Ident,
) -> TokenStream {
let codec_name = syn::parse_str::<syn::Path>(T::CODEC_PATH).unwrap();
let codec_name = syn::parse_str::<syn::Path>(method.codec_path()).unwrap();
let service_ident = quote::format_ident!("{}Svc", method.identifier());