Update python and c++ protobuf examples to demonstrate dynamic FileDe…

…scriptorSet generation (foxglove#269)

**Public-Facing Changes**
Updated example Python Protobuf server to demonstrate dynamic generation of FileDescriptorSet schemas.
Updated example C++ server to use Protobuf instead of JSON, including dynamic generation of FileDescriptorSet schemas.

**Description**
Resolves foxglove#255
Resolves foxglove#138
Relates to foxglove/mcap#694

Checks in generated files for Python because we have not yet published them in a Python package (foxglove/schemas#71).

.gitattributes

@@ -1 +1,2 @@
-python/**/proto/*.bin binary linguist-generated
+python/**/proto/**/*.py* linguist-generated
+cpp/**/proto/**/*.proto linguist-generated

.vscode/settings.json

@@ -20,6 +20,12 @@
  "python.analysis.typeCheckingMode": "strict",
  "jest.rootPath": "typescript",
 
+ "search.exclude": {
+ "**/*_pb2.py*": true,
+ "**/*.pb.cc": true,
+ "**/*.pb.h": true
+ },
+
  // https://github.com/microsoft/vscode-cpptools/issues/722
  "C_Cpp.autoAddFileAssociations": false
 }

cpp/.clang-format

@@ -28,3 +28,6 @@ IncludeCategories:
  - Regex: "^<.*"
  Priority: 3
  SortPriority: 0
+ - Regex: ".*"
+ Priority: 4
+ SortPriority: 0

cpp/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:focal AS base
+FROM ubuntu:jammy AS base
 
 # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
 ENV DEBIAN_FRONTEND=noninteractive
@@ -12,40 +12,31 @@ RUN apt-get update && \
  make \
  perl \
  python3 \
- python3-pip 
+ python3-pip \
+ clang \
+ clang-format
 
-
-RUN echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" >> /etc/apt/sources.list && \
- curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - &&\
- apt-get update && \
- apt-get install -y --no-install-recommends --no-install-suggests \
- clang-13 \
- clang-format-13 
-
-RUN update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100
-RUN update-alternatives --install /usr/bin/git-clang-format git-clang-format /usr/bin/git-clang-format-13 100
-
-ENV CC=clang-13
-ENV CXX=clang++-13
+ENV CC=clang
+ENV CXX=clang++
 
 WORKDIR /src
 
 FROM base as build
 RUN pip --no-cache-dir install conan
 
-ENV CONAN_V2_MODE=1
 RUN conan config init
 RUN conan profile update settings.compiler.cppstd=17 default
 
 FROM build as build_example_server
-COPY ./examples /src/examples/
+COPY ./examples/conanfile.py /src/examples/conanfile.py
 COPY ./foxglove-websocket /src/foxglove-websocket/
 COPY ./.clang-format /src/
 RUN conan editable add ./foxglove-websocket foxglove-websocket/0.0.1
-RUN conan install examples --install-folder examples/build --build=openssl --build=zlib
+RUN conan install examples --install-folder examples/build --build=openssl --build=zlib --build=protobuf
 
 FROM build_example_server AS example_server
 COPY --from=build_example_server /src /src
+COPY ./examples /src/examples
 COPY --from=build_example_server /src/examples/build/ /src/examples/build/
 RUN conan build examples --build-folder examples/build
 ENTRYPOINT ["examples/build/bin/example_server"]

cpp/examples/CMakeLists.txt

@@ -4,5 +4,23 @@ project(FoxgloveWebSocketExamples CXX)
 include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
 conan_basic_setup()
 
-add_executable(example_server example_server.cpp)
+find_package(Protobuf 3 REQUIRED)
+
+FILE(GLOB all_protos "proto/foxglove/*.proto")
+
+FOREACH(f ${all_protos})
+ file(RELATIVE_PATH f ${CMAKE_CURRENT_SOURCE_DIR}/proto ${f})
+ STRING(REGEX REPLACE "\\.proto$" "" f ${f})
+ LIST(APPEND proto_sources "autogenerated/${f}.pb.h")
+ LIST(APPEND proto_sources "autogenerated/${f}.pb.cc")
+ENDFOREACH(f)
+
+add_custom_command(
+ OUTPUT ${proto_sources}
+ COMMAND ${CMAKE_COMMAND} -E make_directory autogenerated
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE} --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=autogenerated ${all_protos}
+)
+
+add_executable(example_server example_server.cpp ${proto_sources})
+target_include_directories(example_server PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/autogenerated)
 target_link_libraries(example_server ${CONAN_LIBS})

cpp/examples/conanfile.py

@@ -4,7 +4,7 @@
 class FoxgloveWebSocketExamplesConan(ConanFile):
  settings = "os", "compiler", "build_type", "arch"
  generators = "cmake"
- requires = "foxglove-websocket/0.0.1"
+ requires = "foxglove-websocket/0.0.1", "protobuf/3.21.1"
 
  def build(self):
  cmake = CMake(self)

cpp/examples/example_server.cpp

@@ -1,39 +1,95 @@
 #include <foxglove/websocket/server.hpp>
 
-#include <nlohmann/json.hpp>
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/util/time_util.h>
 
 #include <atomic>
 #include <chrono>
 #include <iostream>
 #include <thread>
 
-using json = nlohmann::json;
+#include "foxglove/SceneUpdate.pb.h"
 
 static uint64_t nanosecondsSinceEpoch() {
  return uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(
  std::chrono::system_clock::now().time_since_epoch())
  .count());
 }
 
+// Adapted from:
+// https://gist.github.com/tomykaira/f0fd86b6c73063283afe550bc5d77594
+// https://github.com/protocolbuffers/protobuf/blob/01fe22219a0312b178a265e75fe35422ea6afbb1/src/google/protobuf/compiler/csharp/csharp_helpers.cc
+static std::string Base64Encode(std::string_view data) {
+ constexpr const char ALPHABET[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ std::string result;
+ // Every 3 bytes of data yields 4 bytes of output
+ result.reserve((data.size() + (3 - 1 /* round up */)) / 3 * 4);
+
+ size_t i = 0;
+ for (; i + 2 < data.size(); i += 3) {
+ result.push_back(ALPHABET[data[i] >> 2]);
+ result.push_back(ALPHABET[((data[i] & 0b11) << 4) | (data[i + 1] >> 4)]);
+ result.push_back(ALPHABET[((data[i + 1] & 0b1111) << 2) | (data[i + 2] >> 6)]);
+ result.push_back(ALPHABET[data[i + 2] & 0b111111]);
+ }
+ switch (data.size() - i) {
+ case 2:
+ result.push_back(ALPHABET[data[i] >> 2]);
+ result.push_back(ALPHABET[((data[i] & 0b11) << 4) | (data[i + 1] >> 4)]);
+ result.push_back(ALPHABET[(data[i + 1] & 0b1111) << 2]);
+ result.push_back('=');
+ break;
+ case 1:
+ result.push_back(ALPHABET[data[i] >> 2]);
+ result.push_back(ALPHABET[(data[i] & 0b11) << 4]);
+ result.push_back('=');
+ result.push_back('=');
+ break;
+ }
+
+ return result;
+}
+
+// Writes the FileDescriptor of this descriptor and all transitive dependencies
+// to a string, for use as a channel schema.
+static std::string SerializeFdSet(const google::protobuf::Descriptor* toplevelDescriptor) {
+ google::protobuf::FileDescriptorSet fdSet;
+ std::queue<const google::protobuf::FileDescriptor*> toAdd;
+ toAdd.push(toplevelDescriptor->file());
+ std::unordered_set<std::string> seenDependencies;
+ while (!toAdd.empty()) {
+ const google::protobuf::FileDescriptor* next = toAdd.front();
+ toAdd.pop();
+ next->CopyTo(fdSet.add_file());
+ for (int i = 0; i < next->dependency_count(); ++i) {
+ const auto& dep = next->dependency(i);
+ if (seenDependencies.find(dep->name()) == seenDependencies.end()) {
+ seenDependencies.insert(dep->name());
+ toAdd.push(dep);
+ }
+ }
+ }
+ return fdSet.SerializeAsString();
+}
+
+// https://danceswithcode.net/engineeringnotes/quaternions/quaternions.html
+static void setAxisAngle(foxglove::Quaternion* q, double x, double y, double z, double angle) {
+ double s = std::sin(angle / 2);
+ q->set_x(x * s);
+ q->set_y(y * s);
+ q->set_z(z * s);
+ q->set_w(std::cos(angle / 2));
+}
+
 int main() {
- foxglove::websocket::Server server{8765, "example server"};
+ foxglove::websocket::Server server{8765, "C++ Protobuf example server"};
 
  const auto chanId = server.addChannel({
  .topic = "example_msg",
- .encoding = "json",
- .schemaName = "ExampleMsg",
- .schema =
- json{
- {"type", "object"},
- {
- "properties",
- {
- {"msg", {{"type", "string"}}},
- {"count", {{"type", "number"}}},
- },
- },
- }
- .dump(),
+ .encoding = "protobuf",
+ .schemaName = foxglove::SceneUpdate::descriptor()->full_name(),
+ .schema = Base64Encode(SerializeFdSet(foxglove::SceneUpdate::descriptor())),
  });
 
  server.setSubscribeHandler([&](foxglove::websocket::ChannelId chanId) {
@@ -43,16 +99,38 @@ int main() {
  std::cout << "last client unsubscribed from " << chanId << std::endl;
  });
 
- uint64_t i = 0;
  std::shared_ptr<asio::steady_timer> timer;
  std::function<void()> setTimer = [&] {
- timer = server.getEndpoint().set_timer(200, [&](std::error_code const& ec) {
+ timer = server.getEndpoint().set_timer(50, [&](std::error_code const& ec) {
  if (ec) {
  std::cerr << "timer error: " << ec.message() << std::endl;
  return;
  }
- server.sendMessage(chanId, nanosecondsSinceEpoch(),
- json{{"msg", "Hello"}, {"count", i++}}.dump());
+
+ const auto now = nanosecondsSinceEpoch();
+
+ foxglove::SceneUpdate msg;
+ auto* entity = msg.add_entities();
+ *entity->mutable_timestamp() = google::protobuf::util::TimeUtil::NanosecondsToTimestamp(now);
+ entity->set_frame_id("root");
+ auto* cube = entity->add_cubes();
+ auto* size = cube->mutable_size();
+ size->set_x(1);
+ size->set_y(1);
+ size->set_z(1);
+ auto* position = cube->mutable_pose()->mutable_position();
+ position->set_x(2);
+ position->set_y(0);
+ position->set_z(0);
+ auto* orientation = cube->mutable_pose()->mutable_orientation();
+ setAxisAngle(orientation, 0, 0, 1, double(now) / 1e9 * 0.5);
+ auto* color = cube->mutable_color();
+ color->set_r(0.6);
+ color->set_g(0.2);
+ color->set_b(1);
+ color->set_a(1);
+
+ server.sendMessage(chanId, now, msg.SerializeAsString());
  setTimer();
  });
  };