Vendor Generated Protobuf Code (apache#3947) (apache#3950)

* Vendor generated protobuf code (apache#3947)

* RAT

* Fix build without json

* Review feedback

* Doc tweak

* Fix Arch install instructions

.gitattributes

@@ -0,0 +1,2 @@
+datafusion/proto/src/generated/prost.rs linguist-generated
+datafusion/proto/src/generated/pbjson.rs linguist-generated

.github/workflows/rust.yml

@@ -81,16 +81,6 @@ jobs:
  - uses: actions/checkout@v3
  with:
  submodules: true
- - name: Install protobuf compiler
- shell: bash
- run: |
- mkdir -p $HOME/d/protoc
- cd $HOME/d/protoc
- export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
- curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
- unzip $PROTO_ZIP
- export PATH=$PATH:$HOME/d/protoc/bin
- protoc --version
  - name: Cache Cargo
  uses: actions/cache@v3
  with:
@@ -124,7 +114,8 @@ jobs:
  cargo run --example rewrite_expr
  cargo run --example simple_udf
  cargo run --example simple_udaf
-
+ - name: Verify Working Directory Clean
+ run: git diff --exit-code
 
  integration-test:
  name: "Compare to postgres"

datafusion/proto/Cargo.toml

@@ -28,16 +28,16 @@ keywords = ["arrow", "query", "sql"]
 edition = "2021"
 rust-version = "1.62"
 
-[package.metadata.docs.rs]
-rustc-args = ["--cfg", "docsrs"]
+# Exclude proto files so crates.io consumers don't need protoc
+exclude = ["*.proto"]
 
 [lib]
 name = "datafusion_proto"
 path = "src/lib.rs"
 
 [features]
 default = []
-json = ["pbjson", "pbjson-build", "serde", "serde_json"]
+json = ["pbjson", "serde", "serde_json"]
 
 [dependencies]
 arrow = "25.0.0"
@@ -55,5 +55,5 @@ doc-comment = "0.3"
 tokio = "1.18"
 
 [build-dependencies]
-pbjson-build = { version = "0.5", optional = true }
+pbjson-build = { version = "0.5" }
 prost-build = { version = "0.11.1" }

datafusion/proto/build.rs

@@ -15,25 +15,29 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::path::{Path, PathBuf};
+
 type Error = Box<dyn std::error::Error>;
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 fn main() -> Result<(), String> {
  // for use in docker build where file changes can be wonky
  println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
- println!("cargo:rerun-if-changed=proto/datafusion.proto");
 
- build()?;
+ // We don't include the proto files in releases so that downstreams
+ // do not need to have PROTOC included
+ if Path::new("proto/datafusion.proto").exists() {
+ println!("cargo:rerun-if-changed=proto/datafusion.proto");
+ build()?
+ }
 
  Ok(())
 }
 
 fn build() -> Result<(), String> {
- use std::io::Write;
-
- let out = std::path::PathBuf::from(
- std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment variable"),
- );
+ let out: PathBuf = std::env::var("OUT_DIR")
+ .expect("Cannot find OUT_DIR environment variable")
+ .into();
  let descriptor_path = out.join("proto_descriptor.bin");
 
  prost_build::Config::new()
@@ -43,11 +47,9 @@ fn build() -> Result<(), String> {
  .compile_protos(&["proto/datafusion.proto"], &["proto"])
  .map_err(|e| format!("protobuf compilation failed: {}", e))?;
 
- #[cfg(feature = "json")]
  let descriptor_set = std::fs::read(&descriptor_path)
  .expect(&*format!("Cannot read {:?}", &descriptor_path));
 
- #[cfg(feature = "json")]
  pbjson_build::Builder::new()
  .register_descriptors(&descriptor_set)
  .expect(&*format!(
@@ -57,27 +59,11 @@ fn build() -> Result<(), String> {
  .build(&[".datafusion"])
  .map_err(|e| format!("pbjson compilation failed: {}", e))?;
 
- // .serde.rs is not a valid package name, so append to datafusion.rs so we can treat it normally
- let proto = std::fs::read_to_string(out.join("datafusion.rs")).unwrap();
-
- #[cfg(feature = "json")]
- let json = std::fs::read_to_string(out.join("datafusion.serde.rs")).unwrap();
-
- #[cfg(feature = "docsrs")]
- let path = out.join("datafusion.rs");
- #[cfg(not(feature = "docsrs"))]
- let path = "src/generated/datafusion.rs";
-
- let mut file = std::fs::OpenOptions::new()
- .write(true)
- .truncate(true)
- .create(true)
- .open(path)
- .unwrap();
- file.write_all(proto.as_str().as_ref()).unwrap();
+ let prost = out.join("datafusion.rs");
+ let pbjson = out.join("datafusion.serde.rs");
 
- #[cfg(feature = "json")]
- file.write_all(json.as_str().as_ref()).unwrap();
+ std::fs::copy(prost, "src/generated/prost.rs").unwrap();
+ std::fs::copy(pbjson, "src/generated/pbjson.rs").unwrap();
 
  Ok(())
 }

datafusion/proto/src/generated/mod.rs

@@ -18,13 +18,9 @@
 #[allow(clippy::all)]
 #[rustfmt::skip]
 #[cfg(not(docsrs))]
-pub mod datafusion;
-
-#[cfg(docsrs)]
-#[allow(clippy::all)]
 pub mod datafusion {
- include!(concat!(env!("OUT_DIR"), "/datafusion.rs"));
+ include!("prost.rs");
 
  #[cfg(feature = "json")]
- include!(concat!(env!("OUT_DIR"), "/datafusion.serde.rs"));
+ include!("pbjson.rs");
 }