diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 341c53d392..9dcadae328 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -19,6 +19,42 @@ concurrency: # https://github.com/actions/runner-images jobs: + flatbuffers-codegen: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + cxx: [g++-11, clang++-14] + steps: + - uses: actions/checkout@v7.0.1 + - name: Install codegen dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake g++-11 clang-14 \ + libprotobuf-dev libprotoc-dev protobuf-compiler \ + libgflags-dev libleveldb-dev libssl-dev zlib1g-dev + - name: Build isolated Gperftools + env: + CXX: ${{ matrix.cxx }} + run: | + archive="$RUNNER_TEMP/gperftools-2.16.tar.gz" + curl --fail --location --retry 3 \ + https://github.com/gperftools/gperftools/releases/download/gperftools-2.16/gperftools-2.16.tar.gz \ + --output "$archive" + printf '%s %s\n' f12624af5c5987f2cc830ee534f754c3c5961eec08004c26a8b80de015cf056f "$archive" | sha256sum --check + tar -xzf "$archive" -C "$RUNNER_TEMP" + cd "$RUNNER_TEMP/gperftools-2.16" + ./configure --prefix="$RUNNER_TEMP/gperftools-install" \ + --enable-frame-pointers --disable-libunwind + make -j2 + make install + printf 'CMAKE_PREFIX_PATH=%s\n' "$RUNNER_TEMP/gperftools-install" >> "$GITHUB_ENV" + - name: Verify FlatBuffers generation and runtime + env: + CXX: ${{ matrix.cxx }} + JOBS: 2 + run: bash tools/flatbuffers/verify.sh "$RUNNER_TEMP/flatbuffers-codegen" + compile-with-make: name: compile-with-make (${{ matrix.name }}) runs-on: ubuntu-22.04 diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index a4e4272afe..0de86024b4 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -18,6 +18,34 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + flatbuffers-codegen: + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + protobuf: [protobuf@21, protobuf@29] + steps: + - uses: actions/checkout@v7.0.1 + - name: Install codegen dependencies + run: | + brew install cmake openssl gflags leveldb snappy gperftools + if [ "${{ matrix.protobuf }}" = protobuf@21 ]; then + # Same local tap as compile-with-make-cmake-protobuf21. + brew tap-new --no-git local/pins + cp homebrew-formula/protobuf@21.rb "$(brew --repository local/pins)/Formula/" + brew install local/pins/protobuf@21 + else + brew install ${{ matrix.protobuf }} + fi + - name: Verify FlatBuffers generation and runtime + env: + CXX: clang++ + JOBS: 2 + run: | + export CMAKE_PREFIX_PATH="$(brew --prefix ${{ matrix.protobuf }});$(brew --prefix)" + export OPENSSL_ROOT_DIR="$(brew --prefix openssl)" + bash tools/flatbuffers/verify.sh "$RUNNER_TEMP/flatbuffers-codegen" + compile-with-make-cmake-protobuf21: runs-on: macos-latest # https://github.com/actions/runner-images diff --git a/BUILD.bazel b/BUILD.bazel index 23e433c76e..5882a034da 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -20,6 +20,12 @@ licenses(["notice"]) # Apache v2 exports_files(["LICENSE"]) +config_setting( + name = "brpc_with_flatbuffers", + define_values = {"BRPC_WITH_FLATBUFFERS": "true"}, + visibility = ["//visibility:public"], +) + COPTS = [ "-fno-omit-frame-pointer", ] + select({ @@ -45,6 +51,9 @@ DEFINES = [ }) + select({ "//bazel/config:brpc_with_thrift": ["ENABLE_THRIFT_FRAMED_PROTOCOL=1"], "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": ["BRPC_WITH_FLATBUFFERS=1"], + "//conditions:default": ["BRPC_WITH_FLATBUFFERS=0"], }) + select({ "//bazel/config:brpc_with_thrift_legacy_version": [], "//conditions:default": ["THRIFT_STDCXX=std"], @@ -125,6 +134,14 @@ genrule( "//conditions:default": "0", }) + """ +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS """ + select({ + ":brpc_with_flatbuffers": "1", + "//conditions:default": "0", + }) + + """ #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif @@ -542,6 +559,8 @@ brpc_proto_library( visibility = ["//visibility:public"], ) +FLATBUFFERS_SRC_PATTERNS = ["src/brpc/flatbuffers/*.cpp"] + URMA_SRC_PATTERNS = [ "src/brpc/urma/*.cpp", "src/brpc/urma/**/*.cpp", @@ -565,7 +584,7 @@ BRPC_BASE_SRCS = glob( "src/brpc/policy/thrift_protocol.cpp", "src/brpc/event_dispatcher_epoll.cpp", "src/brpc/event_dispatcher_kqueue.cpp", - ] + URMA_SRC_PATTERNS, + ] + URMA_SRC_PATTERNS + FLATBUFFERS_SRC_PATTERNS, ) cc_library( @@ -576,6 +595,9 @@ cc_library( "src/brpc/**/thrift*.cpp", ]), "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": glob(FLATBUFFERS_SRC_PATTERNS), + "//conditions:default": [], }) + select({ "//bazel/config:brpc_with_urma_use_real": URMA_SRCS, "//bazel/config:brpc_with_urma": URMA_SRCS + [ @@ -615,6 +637,9 @@ cc_library( "@org_apache_thrift//:thrift", ], "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:runtime_cc"], + "//conditions:default": [], }), ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59acdfe3bc..0966595d13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF) option(WITH_BORINGSSL "With BoringSSL" OFF) option(WITH_DEBUG_SYMBOLS "With debug symbols" ON) option(WITH_THRIFT "With thrift framed protocol supported" OFF) +option(WITH_FLATBUFFERS "With FlatBuffers message support (headers only)" OFF) option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) option(WITH_SNAPPY "With snappy" OFF) option(WITH_RDMA "With RDMA" OFF) diff --git a/MODULE.bazel b/MODULE.bazel index 97862d36f1..abe905655d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -79,3 +79,19 @@ git_repository( remote = 'https://atomgit.com/openeuler/umdk.git', commit = '564ee727a55523d4351a8fb3c94292b388ebb924', # v26.06.0_CAM ) + +# Do not use `bazel_dep(name = 'flatbuffers', version = '25.2.10')` here. +# The BCR module pulls gRPC and JS/Go/Swift rule dependencies for FlatBuffers' +# full upstream build. bRPC only needs runtime_cc and flatc, and gRPC would +# otherwise introduce another BoringSSL version even when BRPC_WITH_FLATBUFFERS +# is false. Keep this archive and checksum in sync with WORKSPACE. +flatbuffers_http_archive = use_repo_rule( + '@bazel_tools//tools/build_defs/repo:http.bzl', + 'http_archive', +) +flatbuffers_http_archive( + name = 'com_github_google_flatbuffers', + sha256 = 'b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421', + strip_prefix = 'flatbuffers-25.2.10', + urls = ['https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz'], +) diff --git a/Makefile b/Makefile index 271b518ae6..08df1f04e9 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,9 @@ JSON2PB_SOURCES = $(foreach d,$(JSON2PB_DIRS),$(wildcard $(addprefix $(d)/*,$(SR JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES))) BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy src/brpc/policy/mysql src/brpc/rdma +ifeq ($(WITH_FLATBUFFERS),1) +BRPC_DIRS += src/brpc/flatbuffers +endif ifeq ($(WITH_URMA),1) BRPC_DIRS += src/brpc/urma endif diff --git a/WORKSPACE b/WORKSPACE index 22fc411b32..9778a704ba 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -150,6 +150,14 @@ http_archive( urls = ["https://github.com/google/crc32c/archive/1.1.2.tar.gz"], ) +# Optional FlatBuffers support uses runtime_cc; keep this version in sync with MODULE.bazel. +http_archive( + name = "com_github_google_flatbuffers", + integrity = "sha256-ucLfSXB8V6SPwJI9UrjHO+ty1nX51EsiEeRWm+QKdCE=", + strip_prefix = "flatbuffers-25.2.10", + urls = ["https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz"], +) + http_archive( name = "com_github_google_glog", # 2021-05-07T23:06:39Z patch_args = ["-p1"], diff --git a/config_brpc.sh b/config_brpc.sh index 0efa4d374d..e940e486f4 100755 --- a/config_brpc.sh +++ b/config_brpc.sh @@ -54,9 +54,10 @@ else LDD=ldd fi -TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` +TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-flatbuffers,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` WITH_GLOG=0 WITH_THRIFT=0 +WITH_FLATBUFFERS=0 WITH_RDMA=0 WITH_URMA=0 URMA_MOCK_MODE=auto @@ -91,6 +92,7 @@ while true; do --cxx ) CXX=$2; shift 2 ;; --with-glog ) WITH_GLOG=1; shift 1 ;; --with-thrift) WITH_THRIFT=1; shift 1 ;; + --with-flatbuffers) WITH_FLATBUFFERS=1; shift 1 ;; --with-rdma) WITH_RDMA=1; shift 1 ;; --with-urma) WITH_URMA=1; shift 1 ;; --with-urma-mock) URMA_MOCK_MODE=on; shift 1 ;; @@ -479,6 +481,7 @@ append_to_output "HDRS=$($ECHO $HDRS)" append_to_output "LIBS=$($ECHO $LIBS)" append_to_output "PROTOC=$PROTOC" append_to_output "PROTOBUF_HDR=$PROTOBUF_HDR" +append_to_output "WITH_FLATBUFFERS=$WITH_FLATBUFFERS" append_to_output "CC=$CC" append_to_output "CXX=$CXX" append_to_output "GCC_VERSION=$GCC_VERSION" @@ -486,7 +489,7 @@ append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS" append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS" # CPP means C PreProcessing, not C PlusPlus -CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" +CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_WITH_FLATBUFFERS=$WITH_FLATBUFFERS -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" # Avoid over-optimizations of TLS variables by GCC>=4.8 # See: https://github.com/apache/brpc/issues/1693 @@ -506,6 +509,12 @@ if [ "$SYSTEM" = "Darwin" ]; then fi fi +if [ $WITH_FLATBUFFERS != 0 ]; then + FLATBUFFERS_HDR=$(find_dir_of_header_or_die flatbuffers/flatbuffers.h) || exit 1 + append_to_output_headers "$FLATBUFFERS_HDR" + print_success "Found FlatBuffers headers: $FLATBUFFERS_HDR" +fi + if [ $WITH_THRIFT != 0 ]; then THRIFT_LIB=$(find_dir_of_lib_or_die thriftnb) THRIFT_HDR=$(find_dir_of_header_or_die thrift/Thrift.h) @@ -692,6 +701,11 @@ cat << EOF > src/butil/config.h #endif #define BRPC_WITH_GLOG $WITH_GLOG +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS $WITH_FLATBUFFERS + #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif @@ -714,6 +728,7 @@ print_info "C++ std: $CXXFLAGS" print_info "System: $SYSTEM" if [ $WITH_GLOG -ne 0 ]; then print_info "With glog: yes"; fi if [ $WITH_THRIFT -ne 0 ]; then print_info "With thrift: yes"; fi +if [ $WITH_FLATBUFFERS -ne 0 ]; then print_info "With FlatBuffers: yes (headers only)"; fi if [ $WITH_RDMA -ne 0 ]; then print_info "With RDMA: yes"; fi if [ $WITH_URMA -ne 0 ]; then print_info "With URMA: yes"; fi if [ $WITH_MESALINK -ne 0 ]; then print_info "With MesaLink: yes"; fi diff --git a/docs/cn/flatbuffers.md b/docs/cn/flatbuffers.md new file mode 100644 index 0000000000..5e58801594 --- /dev/null +++ b/docs/cn/flatbuffers.md @@ -0,0 +1,116 @@ +# FlatBuffers 消息 + +[English version](../en/flatbuffers.md) + +bRPC 支持基于 IOBuf 的 FlatBuffers 消息、构造器和服务描述符，默认不启用。 +消息构造方案源自 [apache/brpc#3196](https://github.com/apache/brpc/pull/3196)。 + +本组件不包含 `fb_rpc` 传输协议，也不改动 `brpc::Channel` 或 `brpc::Server`。 +服务代码生成和进程内分派测试不涉及网络 RPC，也不是性能测试。 + +## 构建 + +运行库只需要 FlatBuffers 头文件，不链接其库；测试还需要版本匹配的 `flatc`。 +版本不一致时应重新生成头文件，不要删除或放宽上游生成代码中的版本断言。 + +假设 GoogleTest 源码位于 `/usr/src/googletest`： + +```sh +cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest +cmake --build build --target brpc_flatbuffers_unittest -j6 +ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure +``` + +非标准安装可设置 `FLATBUFFERS_INCLUDE_DIR`、`FLATBUFFERS_FLATC_EXECUTABLE` +和 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。其他测试依赖与 bRPC 原有配置相同。 + +Make 使用 `config_brpc.sh --with-flatbuffers`，测试时可通过 +`FLATC=/path/to/flatc` 指定生成器。Bazel 使用 +`--define=BRPC_WITH_FLATBUFFERS=true`。 + +运行库使用了 FlatBuffers builder 的内部结构，目前只支持 25.2.10，其他版本会在 +编译期报错。Bzlmod 和 WORKSPACE 使用同一个源码归档，并用校验和锁定内容。 +未采用 `bazel_dep`，是为了避免引入 gRPC、多语言工具以及与 bRPC 冲突的 BoringSSL +依赖。 + +公共头文件位于 `brpc/flatbuffers/`，命名空间为 `brpc::flatbuffers`。 +构造消息时包含 `message.h`；使用服务描述符和接口时包含 `service.h`。`butil/config.h` 中的 +`BRPC_WITH_FLATBUFFERS` 始终是 0 或 1，因此应使用 `#if` 判断。 + +旧版 flatc 2.0.x 可能生成未全限定名称。业务 schema 不要使用 `brpc` 命名空间，也不要依赖 include 顺序。 + +如需使用其他 FlatBuffers 版本，需要修改运行库、生成器和 codegen 测试中的版本检查， +再用该版本完整运行 `tools/flatbuffers/verify.sh`。步骤见生成器 +[README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version)， +其中以 23.5.26 为例，仅适用于 CMake 构建。 + +## 消息构造和所有权 + +先用上游 `flatc --cpp` 生成 `*_generated.h`，再将 +`brpc::flatbuffers::MessageBuilder` 传给生成的 `Create...` 函数。 +调用 `Finish(root)` 后，通过 `ReleaseMessage()` 取得消息。 + +* `ReleaseMessage()` 不复制 payload。返回的 Message 是 move-only 对象，持有 IOBuf block 引用；builder 复用或析构后，消息仍然有效。 +* Message 和 builder 被移动后，源对象仍可复用。移动启用了 shared string 的 builder 会清空去重缓存，但已生成的 offset 不受影响。 +* `Message::CopyFrom` 和 `MergeFrom` 共享同一个引用计数 IOBuf block，不复制 + payload 或 metadata。通过任一别名修改数据，其他别名也会看到变化。有并发读者时， + 写操作必须由外部同步保护。 +* 从普通 `::flatbuffers::FlatBufferBuilder` 导入时，会复制 payload 和 scratch， + 保留未完成 table 的状态。旧缓冲区由原分配器释放；若源 builder 拥有该分配器， + 导入后也会销毁它。导入过程不猜测应使用 `free` 还是 `delete[]`，也不要求 payload + 前有预留空间。 +* 只使用 MessageBuilder 自身提供的 move、swap 和 release 操作。不要通过基类 + cast 转移对象，也不要调用继承来的 raw-buffer release；这类 detached buffer + 会保留成员 allocator 的地址。 +* payload 前预留 64 字节并初始化为零。`reduce_meta_size_and_get_buf` 可以缩短 + 这段空间；扩大时返回失败，原对象保持不变。缩短 metadata 不会移动或改写 payload。 +* 序列化 const Message 时，输出 IOBuf 会继续引用原存储。它不是 copy-on-write；仍有读者或序列化结果存活时，不要修改 payload 或 metadata。 +* 分配长度在转成 SingleIOBuf 的 uint32_t 长度前会检查。SlabAllocator/builder + 的分配失败会终止进程，release build 也一样，不受 `crash_on_fatal_log` 影响。 + 这是因为 FlatBuffers 的 `vector_downward` 无法在分配器返回 null 后继续工作。 + 解析时复制数据所需的内存若分配失败，则返回 false，保留原 Message。 + +`ParseFbFromIOBUF` 负责检查长度和 framing，解析后的 Message 持有自己的存储引用， +但该接口不负责限制接收消息大小。若 `msg_size` 来自不可信对端，调用方必须先按 +自己的 max-message-size 策略检查，再调用解析接口。分片或未对齐的数据会在 +schema 校验前复制到新存储，因此大小限制必须放在解析之前。 + +连续输入的 payload 地址按 64 字节对齐时，解析可以共享原存储。MessageBuilder +的分配按 64 字节对齐，但最终 payload 只保证满足 schema 的对齐要求，所以本地 +构造的消息未必都能在接收侧零拷贝。当前不支持超过 64 字节的对齐要求。 + +**Framing 检查不等于 schema 校验。** 读取不可信数据前，先调用 +`msg.Verify()`，成功后再使用 `GetRoot()` 或 +`GetMutableRoot()`。合法消息中的可选 string/vector 仍可能为 null。 +Framing 检查失败不会修改原 Message。 + +## Service ID 和代码生成 + +`BrpcDescriptorTable` 保存 namespace、service 名称、按空白分隔的方法名和显式 +方法 ID。ID 必须是唯一的非负 int32。手写 descriptor 可以传空 ID 列表，此时按 +声明顺序分配 ordinal ID；生成服务必须显式声明 ID： + +```fbs +rpc_service BenchmarkService { + First(Request):Response (id: 2); + Second(Request):Response (id: 5); +} +``` + +* `descriptor.method(position)` 按声明顺序取方法。 +* `method.index()` 是稳定的 wire ID，不是数组下标。 +* 稀疏 ID 必须通过 `descriptor.FindMethodByIndex(id)` 查找。传输层不能直接用 wire ID 索引稠密数组。 +* 删除方法后，不要把原 ID 分配给其他方法。调整声明顺序也不应改变已有方法的 ID。 +* namespace `a.b` 和 `a.b.` 会规范化为同一个 service 名称；空 namespace 表示 + 全局作用域。方法全名包含 service 名称。service hash 使用规范化全名和 + MurmurHash3 seed 1。若 ID 需要持久化或在线路上传输，service 名称必须保持稳定。 +* Descriptor 初始化成功后不能再次初始化。它通过 RAII 持有 method；生成的 accessor 使用函数局部静态对象，初始化过程是线程安全的。 + +`tools/flatbuffers/` 中的生成器使用上游 parser 生成服务绑定，并单独链接 +`libflatbuffers`。命令和限制见其 [README](../../tools/flatbuffers/README.md)。 + +生成的 dispatch 会验证请求，拒绝未知、不属于当前 service 或尚未实现的方法。 +失败时，它会执行非空 completion callback；成功分派后，completion 由业务实现负责， +并且必须恰好执行一次。 diff --git a/docs/cn/getting_started.md b/docs/cn/getting_started.md index 2a3f98d166..391873127d 100644 --- a/docs/cn/getting_started.md +++ b/docs/cn/getting_started.md @@ -453,6 +453,10 @@ bRPC默认**不**链接 [libunwind](https://github.com/libunwind/libunwind)。 建议使用最新版本的libunwind。 +## flatbuffers: 25.2.10 + +可选依赖，默认不启用。bRPC 使用了 FlatBuffers builder 的内部接口，只接受该版本；使用其他版本见 [FlatBuffers 消息](flatbuffers.md)。 + # 实例追踪 我们提供了一个程序去帮助你追踪和监控所有brpc实例。 只需要在某处运行 [trackme_server](https://github.com/apache/brpc/tree/master/tools/trackme_server/) 然后再带着 -trackme_server=SERVER参数启动需要被追踪的实例。trackme_server将从实例周期性地收到ping消息然后打印日志。您可以从日志中聚合实例地址，并调用实例的内置服务以获取更多信息。 diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md new file mode 100644 index 0000000000..50748900ad --- /dev/null +++ b/docs/en/flatbuffers.md @@ -0,0 +1,137 @@ +# FlatBuffers messages + +[中文版](../cn/flatbuffers.md) + +bRPC supports IOBuf-backed FlatBuffers messages, builders, and service +descriptors, disabled by default. Message construction is based on +[apache/brpc#3196](https://github.com/apache/brpc/pull/3196). + +This component does not include `fb_rpc` transport or change `brpc::Channel` +or `brpc::Server`. Service generation and in-process dispatch tests do not +exercise network RPC or measure performance. + +## Build + +The runtime needs only FlatBuffers headers; tests also need a matching `flatc`. +Regenerate incompatible headers rather than removing or weakening upstream +version assertions. + +With GoogleTest sources at `/usr/src/googletest`: + +```sh +cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest +cmake --build build --target brpc_flatbuffers_unittest -j6 +ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure +``` + +For nonstandard installations, set `FLATBUFFERS_INCLUDE_DIR`, +`FLATBUFFERS_FLATC_EXECUTABLE`, and `BRPC_SYSTEM_GTEST_SOURCE_DIR` as needed. +The usual bRPC test dependencies still apply. + +For Make, use `config_brpc.sh --with-flatbuffers` and set `FLATC=/path/to/flatc` +when testing. For Bazel, use `--define=BRPC_WITH_FLATBUFFERS=true`. + +The runtime depends on FlatBuffers builder internals and supports only 25.2.10; +other versions fail at compile time. Bzlmod and WORKSPACE use the same +checksum-pinned archive. Using this instead of `bazel_dep` avoids gRPC, +other language tools, and conflicting BoringSSL dependencies. + +Public headers are in `brpc/flatbuffers/`, with namespace `brpc::flatbuffers`. +Include `message.h` for message construction and `service.h` for service +descriptors and interfaces. `BRPC_WITH_FLATBUFFERS` in `butil/config.h` is +0 or 1; test it with `#if`. + +Older flatc 2.0.x may emit unqualified names. Keep business schemas outside +the `brpc` namespace and do not rely on include order. + +To use another FlatBuffers version, update the version checks in the runtime, +the generator and the codegen tests, then run `tools/flatbuffers/verify.sh` +against that version. See the generator +[README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version) +for the steps, using 23.5.26 as an example. They apply to CMake builds only. + +## Message construction and ownership + +Generate `*_generated.h` with upstream `flatc --cpp`. Pass a +`brpc::flatbuffers::MessageBuilder` to the generated `Create...` functions, +then call `Finish(root)` and `ReleaseMessage()`. + +* `ReleaseMessage()` returns a move-only Message without copying payload bytes. + Its IOBuf block reference survives builder reuse or destruction. +* Moves leave the source Message or builder reusable. Moving a shared-string + builder clears its deduplication cache but preserves existing offsets. +* `Message::CopyFrom` and `MergeFrom` share a ref-counted IOBuf block without + copying payload or metadata. Changes through one alias affect all others; + concurrent reads and writes require external synchronization. +* Importing a regular `::flatbuffers::FlatBufferBuilder` copies payload and + scratch, preserving unfinished tables. The original allocator frees the old + storage and is destroyed if owned by the source builder. Import neither + guesses between `free` and `delete[]` nor requires a payload prefix. +* Use only MessageBuilder's own move, swap, and release operations. Base-class + transfers and inherited raw-buffer releases can leave a detached buffer + pointing to the member allocator. +* Released payloads have a 64-byte zeroed prefix. `reduce_meta_size_and_get_buf` + can shrink it without changing the payload address or bytes. Attempts to grow + it fail without changing the message. +* Serializing a const Message shares its storage with the output IOBuf; it is + not copy-on-write. Do not modify payload or metadata while readers or + serialized buffers still use it. +* Sizes are checked before conversion to SingleIOBuf's uint32_t length. + Allocation failure in SlabAllocator or a builder terminates the process even + in release builds, regardless of `crash_on_fatal_log`: FlatBuffers' + `vector_downward` cannot continue after a null allocation. Allocation failure + when copying during parsing returns false and leaves the Message unchanged. + +`ParseFbFromIOBUF` checks lengths and framing and retains a storage reference; +it does not limit received message sizes. Bound untrusted `msg_size` with your +max-message-size policy before parsing: fragmented or unaligned input is copied +to new storage before schema validation. + +Parsing shares contiguous input when its payload is 64-byte aligned. Builder +allocations have this alignment, but finished payloads need only meet their +schema's alignment, so local messages may still require a copy. Alignment +requirements above 64 bytes are unsupported. + +**Framing is not schema verification.** Before reading untrusted data, call +`msg.Verify()` and use `GetRoot()` or +`GetMutableRoot()` only if it succeeds. Optional strings/vectors can +still be null in valid messages. Failed framing checks leave the Message intact. + +## Service IDs and generation + +`BrpcDescriptorTable` holds a namespace, service name, whitespace-separated +method names, and unique nonnegative int32 IDs. Handwritten descriptors may +use an empty ID list to assign IDs in declaration order; generated services +require explicit IDs: + +```fbs +rpc_service BenchmarkService { + First(Request):Response (id: 2); + Second(Request):Response (id: 5); +} +``` + +* `descriptor.method(position)` returns methods in declaration order. +* `method.index()` is the stable wire ID, not an array index. +* Look up sparse IDs with `descriptor.FindMethodByIndex(id)`, not by indexing + a dense array with a wire ID. +* Do not reuse removed method IDs or change existing IDs when reordering + methods. +* Namespaces `a.b` and `a.b.` normalize to the same service name; an empty + namespace means global scope. Method full names include the service name. + Service hashes use the normalized full name and MurmurHash3 seed 1. Keep + service names stable when persisting or transmitting IDs. +* Descriptors own methods through RAII and cannot be reinitialized after + success. + Generated accessors use thread-safe function-local static initialization. + +The generator in `tools/flatbuffers/` uses the upstream parser and links +`libflatbuffers` separately. See its [README](../../tools/flatbuffers/README.md) +for commands and limitations. + +Generated dispatch validates requests and rejects unknown, foreign, or +unimplemented methods. On failure it runs a non-null completion callback. After +successful dispatch, the application owns completion and must run it exactly +once. diff --git a/docs/en/getting_started.md b/docs/en/getting_started.md index cabd5ecc3c..2ac22c3a75 100644 --- a/docs/en/getting_started.md +++ b/docs/en/getting_started.md @@ -454,6 +454,10 @@ It is recommended to use the latest possible version of libunwind. no known issues. +## flatbuffers: 25.2.10 + +Optional and disabled by default. bRPC uses FlatBuffers builder internals, so only this version is accepted; see [FlatBuffers messages](flatbuffers.md) for using another version. + # Track instances We provide a program to help you to track and monitor all brpc instances. Just run [trackme_server](https://github.com/apache/brpc/tree/master/tools/trackme_server/) somewhere and launch need-to-be-tracked instances with -trackme_server=SERVER. The trackme_server will receive pings from instances periodically and print logs when it does. You can aggregate instance addresses from the log and call builtin services of the instances for further information. diff --git a/src/brpc/CMakeLists.txt b/src/brpc/CMakeLists.txt index 4efd059bfd..03e45da465 100644 --- a/src/brpc/CMakeLists.txt +++ b/src/brpc/CMakeLists.txt @@ -43,6 +43,14 @@ set(BRPC_DEFINITIONS BRPC_WITH_URMA=${WITH_URMA_VAL} BRPC_WITH_UBRING=${WITH_UBRING_VAL}) set(BRPC_INCLUDE_DIRS) +if(WITH_FLATBUFFERS) + find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) + if(NOT FLATBUFFERS_INCLUDE_DIR) + message(FATAL_ERROR + "WITH_FLATBUFFERS requires FlatBuffers headers; set FLATBUFFERS_INCLUDE_DIR.") + endif() + list(APPEND BRPC_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR}) +endif() if(WITH_THRIFT) list(APPEND BRPC_DEFINITIONS ENABLE_THRIFT_FRAMED_PROTOCOL) find_library(THRIFT_LIB NAMES thrift) @@ -241,6 +249,9 @@ endif() file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/thrift*.cpp") +if(NOT WITH_FLATBUFFERS) + list(FILTER BRPC_SOURCES EXCLUDE REGEX "/brpc/flatbuffers/.*\\.cpp$") +endif() file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/event_dispatcher_*.cpp") # When building with the real liburma, exclude the link-time mock so its urma_* diff --git a/src/brpc/flatbuffers/message.cpp b/src/brpc/flatbuffers/message.cpp new file mode 100644 index 0000000000..d3116d9a75 --- /dev/null +++ b/src/brpc/flatbuffers/message.cpp @@ -0,0 +1,310 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/flatbuffers/message.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include "butil/logging.h" + +namespace brpc { +namespace flatbuffers { +namespace { + +uint8_t* AlignPayload(uint8_t* data) { + const uintptr_t address = reinterpret_cast(data); + const size_t padding = (kBufferAlignment - address % kBufferAlignment) % + kBufferAlignment; + return data + padding; +} + +} // namespace + +SlabAllocator& SlabAllocator::operator=(SlabAllocator&& other) noexcept { + if (this != &other) { + SlabAllocator tmp(std::move(other)); + swap(tmp); + } + return *this; +} + +void SlabAllocator::swap(SlabAllocator& other) noexcept { + _iobuf.swap(other._iobuf); + std::swap(_data, other._data); + std::swap(_capacity, other._capacity); +} + +uint8_t* SlabAllocator::allocate(size_t size) { + RELEASE_ASSERT_VERBOSE(size > 0 && size < FLATBUFFERS_MAX_BUFFER_SIZE, + "Invalid FlatBuffers allocation size"); + RELEASE_ASSERT_VERBOSE(_data == nullptr, + "Allocator already has a live buffer"); + butil::SingleIOBuf storage; + const uint32_t allocation_size = static_cast( + size + kDefaultMetaSize + kBufferAlignment - 1); + uint8_t* raw = static_cast(storage.allocate(allocation_size)); + RELEASE_ASSERT_VERBOSE(raw != nullptr, + "Fail to allocate FlatBuffers storage"); + _data = AlignPayload(raw + kDefaultMetaSize); + _capacity = size; + _iobuf.swap(storage); + return _data; +} + +void SlabAllocator::deallocate(uint8_t* p, size_t /*size*/) { + if (!p) { + return; + } + RELEASE_ASSERT_VERBOSE(p == _data, + "Invalid FlatBuffers deallocation pointer"); + _iobuf.reset(); + _data = nullptr; + _capacity = 0; +} + +uint8_t* SlabAllocator::reallocate_downward( + uint8_t* old_p, size_t old_size, size_t new_size, + size_t in_use_back, size_t in_use_front) { + RELEASE_ASSERT_VERBOSE(old_p != nullptr && old_p == _data && + old_size == _capacity, + "Invalid FlatBuffers reallocation buffer"); + RELEASE_ASSERT_VERBOSE(new_size > old_size, + "FlatBuffers reallocation must grow"); + RELEASE_ASSERT_VERBOSE(in_use_back <= old_size && + in_use_front <= old_size - in_use_back, + "Invalid FlatBuffers scratch or payload size"); + // Keep the old allocation alive until BOTH data and scratch are copied. + SlabAllocator replacement; + uint8_t* data = replacement.allocate(new_size); + if (in_use_back) { + memcpy(data + new_size - in_use_back, + old_p + old_size - in_use_back, in_use_back); + } + if (in_use_front) { + memcpy(data, old_p, in_use_front); + } + swap(replacement); + return _data; +} + +Message::Message(const butil::IOBuf::BlockRef& ref, uint32_t meta_size, + uint32_t msg_size) + : _iobuf(ref), _meta_size(meta_size), _msg_size(msg_size) {} + +Message& Message::operator=(Message&& other) noexcept { + if (this != &other) { + Clear(); + Swap(other); + } + return *this; +} + +void Message::Swap(Message& other) noexcept { + _iobuf.swap(other._iobuf); + std::swap(_meta_size, other._meta_size); + std::swap(_msg_size, other._msg_size); +} + +void Message::MergeFrom(const Message& other) { + if (&other == this) { + return; + } + _iobuf = other._iobuf; + _meta_size = other._meta_size; + _msg_size = other._msg_size; +} + +void Message::Clear() { + _iobuf.reset(); + _meta_size = 0; + _msg_size = 0; +} + +const uint8_t* Message::data() const { + const uint8_t* raw = static_cast(_iobuf.get_begin()); + return raw ? raw + _meta_size : nullptr; +} + +bool Message::parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, + size_t meta_size) { + const size_t total = buf.size(); + // Subtraction avoids overflow, and the limits cover SingleIOBuf's uint32_t + // sizes, including allocation overhead, before any narrowing conversion. + if (msg_size == 0 || total >= FLATBUFFERS_MAX_BUFFER_SIZE || + meta_size > total || msg_size != total - meta_size) { + return false; + } + butil::SingleIOBuf storage; + const butil::StringPiece first = buf.backing_block(0); + if (first.size() == total && + reinterpret_cast(first.data() + meta_size) % + kBufferAlignment == 0) { + if (!storage.assign(buf, static_cast(total))) { + return false; + } + } else { + uint8_t* raw = static_cast(storage.allocate( + static_cast(total + kBufferAlignment - 1))); + if (!raw) { + return false; + } + uint8_t* begin = AlignPayload(raw + meta_size) - meta_size; + if (buf.copy_to(begin, total) != total) { + return false; + } + const butil::IOBuf::BlockRef& ref = storage.get_cur_ref(); + butil::IOBuf::BlockRef aligned_ref = { + ref.offset + static_cast(begin - raw), + static_cast(total), ref.block}; + butil::SingleIOBuf aligned(aligned_ref); + storage.swap(aligned); + } + _iobuf.swap(storage); + _meta_size = static_cast(meta_size); + _msg_size = static_cast(msg_size); + return true; +} + +bool Message::append_msg_to_iobuf(butil::IOBuf& buf) const { + if (!data() || !_msg_size) { + return false; + } + _iobuf.append_to(&buf); + return true; +} + +void* Message::reduce_meta_size_and_get_buf(uint32_t new_size) { + if (!data() || new_size > _meta_size) { + errno = EINVAL; + return nullptr; + } + if (new_size != _meta_size) { + const uint32_t offset = _meta_size - new_size; + const butil::IOBuf::BlockRef& ref = _iobuf.get_cur_ref(); + butil::IOBuf::BlockRef sub_ref = { + ref.offset + offset, ref.length - offset, ref.block}; + butil::SingleIOBuf slice(sub_ref); + _iobuf.swap(slice); + _meta_size = new_size; + } + return mutable_buf_begin(); +} + +MessageBuilder::MessageBuilder(size_t initial_size) + : ::flatbuffers::FlatBufferBuilder( + initial_size, &slab_allocator_, false, kBufferAlignment) { + RELEASE_ASSERT_VERBOSE(initial_size > 0 && + initial_size < FLATBUFFERS_MAX_BUFFER_SIZE, + "Invalid FlatBuffers initial size"); +} + +MessageBuilder::MessageBuilder(MessageBuilder&& other) : MessageBuilder() { + Swap(other); +} + +MessageBuilder& MessageBuilder::operator=(MessageBuilder&& other) { + if (this != &other) { + MessageBuilder tmp(std::move(other)); + Swap(tmp); + } + return *this; +} + +void MessageBuilder::ClearStringPool() { + // FlatBuffers' shared-string comparator stores a vector_downward pointer. + // Discard only this optional cache when moving between vector objects. + delete string_pool; + string_pool = nullptr; +} + +void MessageBuilder::Swap(MessageBuilder& other) { + if (this == &other) { + return; + } + ClearStringPool(); + other.ClearStringPool(); + slab_allocator_.swap(other.slab_allocator_); + ::flatbuffers::FlatBufferBuilder::Swap(other); + // Each builder must continue to refer to its OWN allocator member. + buf_.swap_allocator(other.buf_); +} + +MessageBuilder::MessageBuilder(::flatbuffers::FlatBufferBuilder&& src) + : ::flatbuffers::FlatBufferBuilder(std::move(src)) { + ClearStringPool(); + RELEASE_ASSERT_VERBOSE(minalign_ <= kBufferAlignment, + "Unsupported FlatBuffers alignment"); + decltype(buf_) replacement( + buf_.capacity() ? buf_.capacity() : 1024, + &slab_allocator_, false, kBufferAlignment); + if (buf_.capacity()) { + replacement.push(buf_.data(), buf_.size()); + for (::flatbuffers::uoffset_t i = 0; i < buf_.scratch_size(); ++i) { + replacement.scratch_push_small(buf_.scratch_data()[i]); + } + } + // The temporary returns the original buffer to its actual allocator. + buf_.swap(replacement); +} + +MessageBuilder& MessageBuilder::operator=( + ::flatbuffers::FlatBufferBuilder&& src) { + if (static_cast<::flatbuffers::FlatBufferBuilder*>(this) != &src) { + MessageBuilder tmp(std::move(src)); + Swap(tmp); + } + return *this; +} + +Message MessageBuilder::ReleaseMessage() { + RELEASE_ASSERT_VERBOSE(finished && buf_.size() > 0, + "Finish the FlatBuffer before releasing a message"); + const uint32_t msg_size = static_cast(buf_.size()); + const uint8_t* msg_data = buf_.data(); + const butil::SingleIOBuf& storage = slab_allocator_._iobuf; + const uint8_t* raw = static_cast(storage.get_begin()); + RELEASE_ASSERT_VERBOSE(raw != nullptr, "Missing FlatBuffers storage"); + RELEASE_ASSERT_VERBOSE(msg_data >= raw + kDefaultMetaSize, + "Missing FlatBuffers metadata prefix"); + const size_t begin = msg_data - raw - kDefaultMetaSize; + RELEASE_ASSERT_VERBOSE( + begin + kDefaultMetaSize + msg_size <= storage.get_length(), + "FlatBuffers message exceeds storage"); + const butil::IOBuf::BlockRef& ref = storage.get_cur_ref(); + const butil::IOBuf::BlockRef sub_ref = { + ref.offset + static_cast(begin), + kDefaultMetaSize + msg_size, ref.block}; + Message msg(sub_ref, kDefaultMetaSize, msg_size); + // Never expose stale scratch or allocator bytes in the reserved prefix. + memset(msg.mutable_buf_begin(), 0, kDefaultMetaSize); + Reset(); + return msg; +} + +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, + size_t meta_size) { + return msg && msg->parse_msg_from_iobuf(buf, msg_size, meta_size); +} + +bool SerializeFbToIOBUF(const Message* msg, butil::IOBuf& buf) { + return msg && msg->append_msg_to_iobuf(buf); +} + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/flatbuffers/message.h b/src/brpc/flatbuffers/message.h new file mode 100644 index 0000000000..7ecbb6c9fc --- /dev/null +++ b/src/brpc/flatbuffers/message.h @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_MESSAGE_H +#define BRPC_FLATBUFFERS_MESSAGE_H + +#include "butil/config.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include + +#if !defined(FLATBUFFERS_VERSION_MAJOR) || \ + !defined(FLATBUFFERS_VERSION_MINOR) || \ + !defined(FLATBUFFERS_VERSION_REVISION) || \ + FLATBUFFERS_VERSION_MAJOR != 25 || \ + FLATBUFFERS_VERSION_MINOR != 2 || \ + FLATBUFFERS_VERSION_REVISION != 10 +#error "bRPC FlatBuffers support requires FlatBuffers 25.2.10" +#endif + +#include "butil/macros.h" +#include "butil/single_iobuf.h" +#include "brpc/nonreflectable_message.h" + +namespace brpc { +namespace flatbuffers { + +// Space preceding a payload, available to a future RPC transport. +constexpr uint32_t kDefaultMetaSize = 64; +// IOBuf slices need not start at an aligned address. The allocator corrects it. +constexpr size_t kBufferAlignment = 64; + +class MessageBuilder; + +// FlatBufferBuilder cannot recover from a null allocation. Allocation failure +// and sizes outside FlatBuffers' offset range are fatal, even in release +// builds. +class SlabAllocator : public ::flatbuffers::Allocator { +public: + SlabAllocator() : _data(nullptr), _capacity(0) {} + SlabAllocator(SlabAllocator&& other) noexcept : SlabAllocator() { + swap(other); + } + SlabAllocator& operator=(SlabAllocator&& other) noexcept; + + uint8_t* allocate(size_t size) override; + void deallocate(uint8_t* p, size_t size) override; + uint8_t* reallocate_downward(uint8_t* old_p, size_t old_size, + size_t new_size, size_t in_use_back, + size_t in_use_front) override; + void swap(SlabAllocator& other) noexcept; + +private: + DISALLOW_COPY_AND_ASSIGN(SlabAllocator); +friend class MessageBuilder; + butil::SingleIOBuf _iobuf; + uint8_t* _data; + size_t _capacity; +}; + +// Construct the allocator before the FlatBufferBuilder base uses it, and +// destroy it after that base has returned its buffer. +struct SlabAllocatorMember { + SlabAllocator slab_allocator_; +}; + +// A move-only message for application code. Protobuf CopyFrom/MergeFrom retain +// the same ref-counted IOBuf instead of deep-copying it. Mutable access through +// any alias changes every message sharing that storage and must not race +// readers. +// Parsing checks framing, not the schema: Verify() must succeed before +// reading data received from an untrusted peer. +class Message : public NonreflectableMessage { +public: + Message() : _meta_size(0), _msg_size(0) {} + Message(Message&& other) noexcept : Message() { Swap(other); } + Message& operator=(Message&& other) noexcept; + + void MergeFrom(const Message&) override; + void Clear() override; + void Swap(Message& other) noexcept; + + const uint8_t* data() const; + // These mutable pointers may refer to storage shared by CopyFrom/MergeFrom. + void* mutable_data() { return const_cast(data()); } + void* mutable_buf_begin() { + return const_cast(_iobuf.get_begin()); + } + void* reduce_meta_size_and_get_buf(uint32_t new_size); + uint32_t get_meta_size() const { return _meta_size; } + size_t size() const { return _msg_size; } + + template + bool Verify() const { + if (!data() || size() < sizeof(::flatbuffers::uoffset_t) || + size() >= FLATBUFFERS_MAX_BUFFER_SIZE) { + return false; + } + ::flatbuffers::Verifier verifier(data(), size()); + return verifier.VerifyBuffer(nullptr); + } + + // These accessors require a schema-verified buffer. GetMutableRoot may + // mutate all Message aliases that retain the same IOBuf block. + template const T* GetRoot() const { + return data() ? ::flatbuffers::GetRoot(data()) : nullptr; + } + template T* GetMutableRoot() { + return data() + ? ::flatbuffers::GetMutableRoot(mutable_data()) : nullptr; + } + + // Failure leaves the old message unchanged. A fragmented or unaligned + // payload is copied into aligned storage; aligned contiguous input is + // shared. + // When msg_size comes from an untrusted peer, callers must bound it with + // their own max-message-size policy before parsing. This method does not + // verify the schema; call Verify() before reading the root. + bool parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, + size_t meta_size); + bool append_msg_to_iobuf(butil::IOBuf& buf) const; + +private: + DISALLOW_COPY_AND_ASSIGN(Message); +friend class MessageBuilder; + Message(const butil::IOBuf::BlockRef& ref, uint32_t meta_size, + uint32_t msg_size); + butil::SingleIOBuf _iobuf; + uint32_t _meta_size; + uint32_t _msg_size; +}; + +class MessageBuilder : private SlabAllocatorMember, + public ::flatbuffers::FlatBufferBuilder { +public: + explicit MessageBuilder(size_t initial_size = 1024); + MessageBuilder(MessageBuilder&& other); + MessageBuilder& operator=(MessageBuilder&& other); + + // Importing a foreign builder copies its payload and scratch data, + // retaining build state. Its original allocator frees the source + // allocation correctly. No free()/delete[] guess or spare-prefix + // assumption is made. + explicit MessageBuilder(::flatbuffers::FlatBufferBuilder&& src); + MessageBuilder& operator=(::flatbuffers::FlatBufferBuilder&& src); + + void Swap(MessageBuilder& other); + // Requires Finish(). The returned message shares storage without copying; + // both this builder and a moved-from builder may immediately be reused. + Message ReleaseMessage(); + +private: + DISALLOW_COPY_AND_ASSIGN(MessageBuilder); + void ClearStringPool(); + // The inherited release methods would retain a pointer to our member + // allocator after this builder dies. Only ReleaseMessage is supported. + using ::flatbuffers::FlatBufferBuilder::Release; + using ::flatbuffers::FlatBufferBuilder::ReleaseRaw; + void ReleaseBufferPointer() = delete; + using ::flatbuffers::FlatBufferBuilder::SwapBufAllocator; +}; + +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, + size_t meta_size = 0); +bool SerializeFbToIOBUF(const Message* msg, butil::IOBuf& buf); + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_FLATBUFFERS_MESSAGE_H diff --git a/src/brpc/flatbuffers/service.cpp b/src/brpc/flatbuffers/service.cpp new file mode 100644 index 0000000000..eb4f44f4e9 --- /dev/null +++ b/src/brpc/flatbuffers/service.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/flatbuffers/service.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include "butil/third_party/murmurhash3/murmurhash3.h" + +namespace brpc { +namespace flatbuffers { +namespace { + +bool IsIdentifier(const std::string& name) { + if (name.empty()) { + return false; + } + for (size_t i = 0; i < name.size(); ++i) { + const char c = name[i]; + if (c != '_' && !(c >= 'a' && c <= 'z') && + !(c >= 'A' && c <= 'Z') && + !(i != 0 && c >= '0' && c <= '9')) { + return false; + } + } + return true; +} + +bool IsNamespace(const std::string& name) { + if (name.empty()) { + return true; + } + size_t begin = 0; + do { + const size_t end = name.find('.', begin); + if (!IsIdentifier(name.substr(begin, end - begin))) { + return false; + } + if (end == std::string::npos) { + return true; + } + begin = end + 1; + } while (begin < name.size()); + return false; +} + +} // namespace + +int ServiceDescriptor::init(const BrpcDescriptorTable& table) { + if (!_methods.empty()) { + errno = EALREADY; + return -1; + } + std::string prefix = table.prefix; + if (!prefix.empty() && prefix.back() == '.') { + prefix.pop_back(); + if (prefix.empty()) { + errno = EINVAL; + return -1; + } + } + if (!IsIdentifier(table.service_name) || !IsNamespace(prefix)) { + errno = EINVAL; + return -1; + } + std::istringstream input(table.method_name_list); + std::vector names; + std::set unique_names; + std::string name; + while (input >> name) { + if (!IsIdentifier(name) || !unique_names.insert(name).second) { + errno = EINVAL; + return -1; + } + names.push_back(name); + } + if (names.empty() || names.size() > static_cast( + std::numeric_limits::max()) || + (!table.method_ids.empty() && table.method_ids.size() != names.size())) { + errno = EINVAL; + return -1; + } + std::set unique_ids; + for (int id : table.method_ids) { + if (id < 0 || !unique_ids.insert(id).second) { + errno = EINVAL; + return -1; + } + } + _name = table.service_name; + _full_name = prefix.empty() ? _name : prefix + "." + _name; + butil::MurmurHash3_x86_32(_full_name.data(), _full_name.size(), 1, &_index); + _methods.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { + const int id = table.method_ids.empty() ? static_cast(i) + : table.method_ids[i]; + _methods.emplace_back(new MethodDescriptor(names[i], this, id)); + } + return 0; +} + +const MethodDescriptor* ServiceDescriptor::method(int position) const { + if (position < 0 || static_cast(position) >= _methods.size()) { + errno = EINVAL; + return nullptr; + } + return _methods[position].get(); +} + +const MethodDescriptor* ServiceDescriptor::FindMethodByIndex(int id) const { + for (const auto& method : _methods) { + if (method->index() == id) { + return method.get(); + } + } + errno = EINVAL; + return nullptr; +} + +MethodDescriptor::MethodDescriptor(const std::string& name, + const ServiceDescriptor* service, int index) + : _name(name), _full_name(service->full_name() + "." + name), + _service(service), _index(index) {} + +int parse_service_descriptors(const BrpcDescriptorTable& table, + ServiceDescriptor** out) { + if (!out) { + errno = EINVAL; + return -1; + } + auto descriptor = std::make_unique(); + if (descriptor->init(table) != 0) { + return -1; + } + *out = descriptor.release(); + return 0; +} + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/flatbuffers/service.h b/src/brpc/flatbuffers/service.h new file mode 100644 index 0000000000..56345f63dd --- /dev/null +++ b/src/brpc/flatbuffers/service.h @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_SERVICE_H +#define BRPC_FLATBUFFERS_SERVICE_H + +#include "butil/config.h" +#include "butil/macros.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include + +namespace google { +namespace protobuf { +class Closure; +class RpcController; +} // namespace protobuf +} // namespace google + +namespace brpc { +namespace flatbuffers { +class Message; +class ServiceDescriptor; + +struct BrpcDescriptorTable { + // A namespace, optionally followed by one period. Empty means global scope. + std::string prefix; + std::string service_name; + // Whitespace-separated method names in declaration order. + std::string method_name_list; + // Stable wire IDs, NOT array positions. Empty retains legacy ordinal IDs. + // New schemas should specify every ID; IDs must be nonnegative and unique. + std::vector method_ids; +}; + +class MethodDescriptor { +public: + MethodDescriptor(const std::string& name, const ServiceDescriptor* service, + int index); + const std::string& name() const { return _name; } + const std::string& full_name() const { return _full_name; } + int index() const { return _index; } + const ServiceDescriptor* service() const { return _service; } + +private: + std::string _name; + std::string _full_name; + const ServiceDescriptor* _service; + int _index; +}; + +// Immutable after successful initialization. Owns all method descriptors. +class ServiceDescriptor { +public: + ServiceDescriptor() : _index(0) {} + int init(const BrpcDescriptorTable& table); + const std::string& name() const { return _name; } + const std::string& full_name() const { return _full_name; } + // MurmurHash3 of the canonical fully-qualified service name, seed 1. + uint32_t index() const { return _index; } + int method_count() const { return static_cast(_methods.size()); } + // Dense declaration-order lookup, for enumeration and generated stubs. + const MethodDescriptor* method(int position) const; + // Sparse stable-ID lookup, for wire dispatch. The fb_rpc transport uses + // this method to resolve stable wire IDs; missing IDs return nullptr. + const MethodDescriptor* FindMethodByIndex(int id) const; + +private: + DISALLOW_COPY_AND_ASSIGN(ServiceDescriptor); + std::string _name; + std::string _full_name; + uint32_t _index; + std::vector > _methods; +}; + +// On success the caller owns *out; on failure *out is unchanged. +int parse_service_descriptors(const BrpcDescriptorTable& table, + ServiceDescriptor** out); + +class RpcChannel { +public: + RpcChannel() = default; + virtual ~RpcChannel() = default; + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) = 0; + +private: + DISALLOW_COPY_AND_ASSIGN(RpcChannel); +}; + +class Service { +public: + Service() = default; + virtual ~Service() = default; + enum ChannelOwnership { STUB_OWNS_CHANNEL, STUB_DOESNT_OWN_CHANNEL }; + virtual const ServiceDescriptor* GetDescriptor() = 0; + // Public dispatch contract consumed by generated services and fb_rpc: + // generated dispatch validates the method and request schema, reports + // failures through controller, and runs a non-null done exactly once. + // Successful implementations may complete asynchronously; request, + // response and controller must remain alive until done is run. + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) = 0; + +private: + DISALLOW_COPY_AND_ASSIGN(Service); +}; + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_FLATBUFFERS_SERVICE_H diff --git a/src/butil/CMakeLists.txt b/src/butil/CMakeLists.txt index 1e57710a19..901bc1954a 100644 --- a/src/butil/CMakeLists.txt +++ b/src/butil/CMakeLists.txt @@ -21,6 +21,12 @@ if(WITH_GLOG) set(BRPC_WITH_GLOG 1) endif() +# Recorded in config.h so installed brpc/flatbuffers headers see the choice. +set(WITH_FLATBUFFERS_VAL "0") +if(WITH_FLATBUFFERS) + set(WITH_FLATBUFFERS_VAL "1") +endif() + set(WITH_CPU_FREQUENCY_VAL "0") if(WITH_CPU_FREQUENCY) set(WITH_CPU_FREQUENCY_VAL "1") diff --git a/src/butil/config.h.in b/src/butil/config.h.in index d8de111be9..ece54504ce 100644 --- a/src/butil/config.h.in +++ b/src/butil/config.h.in @@ -21,6 +21,11 @@ #endif #cmakedefine BRPC_WITH_GLOG @WITH_GLOG_VAL@ +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS @WITH_FLATBUFFERS_VAL@ + #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 1aecc39987..a2c291eb7e 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -229,11 +229,47 @@ root_runfiles( ], ) +FLATBUFFERS_COMPATIBILITY = select({ + "//:brpc_with_flatbuffers": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + +genrule( + name = "flatbuffers_test_messages", + srcs = ["flatbuffers_message.fbs"], + outs = ["flatbuffers_message_generated.h"], + cmd = select({ + "//:brpc_with_flatbuffers": "$(location @com_github_google_flatbuffers//:flatc) --cpp -o $(@D) $(location flatbuffers_message.fbs)", + "//conditions:default": "exit 1", + }), + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + tools = select({ + "//:brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:flatc"], + "//conditions:default": [], + }), +) + +cc_test( + name = "brpc_flatbuffers_unittest", + srcs = [ + "brpc_flatbuffers_unittest.cpp", + ":flatbuffers_test_messages", + ], + copts = COPTS, + includes = ["."], + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + deps = [ + "//:brpc", + "@com_google_googletest//:gtest_main", + ], +) + generate_unittests( name = "brpc_unittests", - srcs = glob([ - "brpc_*_unittest.cpp", - ]), + srcs = glob( + ["brpc_*_unittest.cpp"], + exclude = ["brpc_flatbuffers_unittest.cpp"], + ), deps = [ ":gperftools_helper", "//:brpc", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a0fc2d0536..e2e9a84757 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,21 +59,25 @@ set(BRPC_SYSTEM_GTEST_SOURCE_DIR "" CACHE PATH "System googletest source directo if(DOWNLOAD_GTEST) include(SetupGtest) - set(BRPC_DOWNLOADED_GTEST_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/downloaded-googletest-include) - file(REMOVE_RECURSE ${BRPC_DOWNLOADED_GTEST_INCLUDE_DIR}) - file(MAKE_DIRECTORY ${BRPC_DOWNLOADED_GTEST_INCLUDE_DIR}) - file(COPY - ${PROJECT_BINARY_DIR}/googletest-src/googletest/include/ - ${PROJECT_BINARY_DIR}/googletest-src/googlemock/include/ - DESTINATION ${BRPC_DOWNLOADED_GTEST_INCLUDE_DIR}) - target_include_directories(brpc_test_config BEFORE INTERFACE - ${BRPC_DOWNLOADED_GTEST_INCLUDE_DIR}) + set(BRPC_GTEST_SOURCE_DIR "${PROJECT_BINARY_DIR}/googletest-src") elseif(BRPC_SYSTEM_GTEST_SOURCE_DIR) add_subdirectory("${BRPC_SYSTEM_GTEST_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/system-googletest-build") + set(BRPC_GTEST_SOURCE_DIR "${BRPC_SYSTEM_GTEST_SOURCE_DIR}") else() message(FATAL_ERROR "Googletest is not available") endif() +# Keep the selected headers ahead of unrelated -I paths. The original GTest +# include directories can be marked SYSTEM by its own targets. +set(BRPC_GTEST_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/selected-googletest-include") +file(REMOVE_RECURSE "${BRPC_GTEST_INCLUDE_DIR}") +file(MAKE_DIRECTORY "${BRPC_GTEST_INCLUDE_DIR}") +file(COPY + "${BRPC_GTEST_SOURCE_DIR}/googletest/include/" + "${BRPC_GTEST_SOURCE_DIR}/googlemock/include/" + DESTINATION "${BRPC_GTEST_INCLUDE_DIR}") +target_include_directories(brpc_test_config BEFORE INTERFACE "${BRPC_GTEST_INCLUDE_DIR}") + file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/cert1.key DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/cert2.key DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/cert1.crt DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) @@ -284,8 +288,37 @@ foreach(BTHREAD_UT ${BTHREAD_UNITTESTS}) add_test(NAME ${BTHREAD_UT_WE} COMMAND ${BTHREAD_UT_WE}) endforeach() +# FlatBuffers tests use the release library, independently of the debug suite. +if(WITH_FLATBUFFERS) + find_program(FLATBUFFERS_FLATC_EXECUTABLE NAMES flatc) + if(NOT FLATBUFFERS_FLATC_EXECUTABLE) + message(FATAL_ERROR + "FlatBuffers tests require an installed flatc; set FLATBUFFERS_FLATC_EXECUTABLE.") + endif() + set(FLATBUFFERS_TEST_HEADER + "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_message_generated.h") + add_custom_command( + OUTPUT "${FLATBUFFERS_TEST_HEADER}" + COMMAND "${FLATBUFFERS_FLATC_EXECUTABLE}" --cpp + -o "${CMAKE_CURRENT_BINARY_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/flatbuffers_message.fbs" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/flatbuffers_message.fbs" + "${FLATBUFFERS_FLATC_EXECUTABLE}" + COMMENT "Generating FlatBuffers test messages" + VERBATIM) + add_executable(brpc_flatbuffers_unittest + brpc_flatbuffers_unittest.cpp "${FLATBUFFERS_TEST_HEADER}") + target_include_directories(brpc_flatbuffers_unittest BEFORE PRIVATE + "${BRPC_GTEST_INCLUDE_DIR}") + target_link_libraries(brpc_flatbuffers_unittest PRIVATE + brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) + add_test(NAME brpc_flatbuffers_unittest COMMAND brpc_flatbuffers_unittest) +endif() + # brpc tests file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") +list(REMOVE_ITEM BRPC_UNITTESTS + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp") foreach(BRPC_UT ${BRPC_UNITTESTS}) get_filename_component(BRPC_UT_WE ${BRPC_UT} NAME_WE) add_executable(${BRPC_UT_WE} ${BRPC_UT} $) diff --git a/test/Makefile b/test/Makefile index de4d566bcd..f4d5462e63 100644 --- a/test/Makefile +++ b/test/Makefile @@ -181,12 +181,16 @@ TEST_BTHREAD_SOURCES = $(wildcard bthread_*unittest.cpp) TEST_BTHREAD_OBJS = $(addsuffix .o, $(basename $(TEST_BTHREAD_SOURCES))) TEST_BRPC_SOURCES = $(wildcard brpc_*unittest.cpp) +ifneq ($(WITH_FLATBUFFERS),1) +TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp,$(TEST_BRPC_SOURCES)) +endif TEST_BRPC_OBJS = $(addsuffix .o, $(basename $(TEST_BRPC_SOURCES))) TEST_PROTO_SOURCES = $(wildcard *.proto) TEST_PROTO_OBJS = $(TEST_PROTO_SOURCES:.proto=.pb.o) TEST_BINS = test_butil test_bvar $(TEST_BTHREAD_SOURCES:.cpp=) $(TEST_BRPC_SOURCES:.cpp=) +FLATBUFFERS_TEST_GEN_DIR = ../output/test/flatbuffers-generated .PHONY:all all: $(TEST_BINS) @@ -194,7 +198,7 @@ all: $(TEST_BINS) .PHONY:clean clean:clean_bins @echo "> Cleaning" - rm -rf $(TEST_BUTIL_OBJS) $(TEST_BVAR_OBJS) $(TEST_BTHREAD_OBJS) $(TEST_BRPC_OBJS) $(TEST_PROTO_OBJS) $(TEST_PROTO_SOURCES:.proto=.pb.h) $(TEST_PROTO_SOURCES:.proto=.pb.cc) + rm -rf $(TEST_BUTIL_OBJS) $(TEST_BVAR_OBJS) $(TEST_BTHREAD_OBJS) $(TEST_BRPC_OBJS) $(TEST_PROTO_OBJS) $(TEST_PROTO_SOURCES:.proto=.pb.h) $(TEST_PROTO_SOURCES:.proto=.pb.cc) flatbuffers_message_generated.h $(FLATBUFFERS_TEST_GEN_DIR) $(MAKE) -C.. clean_debug .PHONY:clean_bins @@ -235,6 +239,28 @@ else ifeq ($(SYSTEM),Darwin) $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(GTEST_STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) endif +ifeq ($(WITH_FLATBUFFERS),1) +FLATC ?= flatc +FLATBUFFERS_TEST_HEADER = $(FLATBUFFERS_TEST_GEN_DIR)/flatbuffers_message_generated.h +FLATBUFFERS_TEST_HDRPATHS = -I$(FLATBUFFERS_TEST_GEN_DIR) $(HDRPATHS) + +$(FLATBUFFERS_TEST_HEADER):flatbuffers_message.fbs + @mkdir -p $(FLATBUFFERS_TEST_GEN_DIR) + $(FLATC) --cpp -o $(FLATBUFFERS_TEST_GEN_DIR) $< + +brpc_flatbuffers_unittest.o:brpc_flatbuffers_unittest.cpp $(FLATBUFFERS_TEST_HEADER) | libbrpc.dbg.$(SOEXT) + @echo "> Compiling $@" + $(CXX) -c $(FLATBUFFERS_TEST_HDRPATHS) $(CXXFLAGS) $< -o $@ + +brpc_flatbuffers_unittest:brpc_flatbuffers_unittest.o | libbrpc.dbg.$(SOEXT) + @echo "> Linking $@" +ifeq ($(SYSTEM),Linux) + $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) +else ifeq ($(SYSTEM),Darwin) + $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(GTEST_STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) +endif +endif + brpc_%_unittest:$(TEST_PROTO_OBJS) brpc_%_unittest.o | libbrpc.dbg.$(SOEXT) @echo "> Linking $@" ifeq ($(SYSTEM),Linux) diff --git a/test/brpc_flatbuffers_unittest.cpp b/test/brpc_flatbuffers_unittest.cpp new file mode 100644 index 0000000000..b2b2dd74e0 --- /dev/null +++ b/test/brpc_flatbuffers_unittest.cpp @@ -0,0 +1,565 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "butil/config.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include +#include +#include +#include +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include + +#if !BRPC_WITH_GLOG +namespace logging { +DECLARE_bool(crash_on_fatal_log); +} +#endif + +namespace { +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::SlabAllocator; +using brpc::flatbuffers::ServiceDescriptor; +using brpc::flatbuffers::BrpcDescriptorTable; +using brpc_fbtest::Payload; + +void FinishPayload(::flatbuffers::FlatBufferBuilder& builder, + const std::string& text, int64_t value = 42) { + auto str = builder.CreateString(text); + std::vector numbers = {1, -2, 3, 4000}; + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload(builder, value, str, values)); +} + +Message MakeMessage(size_t length = 16, int64_t value = 42) { + MessageBuilder builder(8); + FinishPayload(builder, std::string(length, 'x'), value); + return builder.ReleaseMessage(); +} + +void ExpectPayload(const Message& msg, size_t length, int64_t value = 42) { + ASSERT_TRUE(msg.Verify()); + const Payload* root = msg.GetRoot(); + ASSERT_NE(nullptr, root); + EXPECT_EQ(value, root->value()); + ASSERT_NE(nullptr, root->message()); + EXPECT_EQ(std::string(length, 'x'), root->message()->str()); + ASSERT_NE(nullptr, root->values()); + ASSERT_EQ(4u, root->values()->size()); + EXPECT_EQ(-2, root->values()->Get(1)); +} + +TEST(FlatbuffersTest, EmptyAndClear) { + Message msg; + EXPECT_EQ(nullptr, msg.data()); + EXPECT_EQ(nullptr, msg.GetRoot()); + EXPECT_EQ(nullptr, msg.GetMutableRoot()); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_FALSE(msg.Verify()); + EXPECT_EQ(0u, msg.size()); + butil::IOBuf wire; + EXPECT_FALSE(brpc::flatbuffers::SerializeFbToIOBUF(&msg, wire)); + EXPECT_FALSE(brpc::flatbuffers::SerializeFbToIOBUF(nullptr, wire)); + EXPECT_FALSE(brpc::flatbuffers::ParseFbFromIOBUF(nullptr, 0, wire)); + msg = MakeMessage(); + msg.Clear(); + msg.Clear(); + EXPECT_EQ(nullptr, msg.data()); + EXPECT_EQ(0u, msg.get_meta_size()); + EXPECT_EQ(0u, msg.size()); +} + +TEST(FlatbuffersTest, ReleaseLifetimeAndGrowth) { + for (size_t length : {0u, 16u, 63u, 1024u, 8192u, 32768u}) { + SCOPED_TRACE(length); + Message msg; + const uint8_t* payload = nullptr; + { + MessageBuilder builder(8); + FinishPayload(builder, std::string(length, 'x')); + payload = builder.GetBufferPointer(); + msg = builder.ReleaseMessage(); + EXPECT_EQ(payload, msg.data()); + EXPECT_EQ(0u, builder.GetSize()); + FinishPayload(builder, "replacement"); + } + ExpectPayload(msg, length); + EXPECT_EQ(std::string(brpc::flatbuffers::kDefaultMetaSize, '\0'), + std::string(static_cast(msg.mutable_buf_begin()), + msg.get_meta_size())); + } +} + +TEST(FlatbuffersTest, MessageMoveAndSwap) { + Message first = MakeMessage(16, 1); + const uint8_t* ptr = first.data(); + Message second(std::move(first)); + EXPECT_EQ(ptr, second.data()); + EXPECT_EQ(nullptr, first.data()); + EXPECT_EQ(0u, first.size()); + EXPECT_EQ(0u, first.get_meta_size()); + EXPECT_FALSE(first.Verify()); + first = MakeMessage(8192, 2); + second = std::move(first); + EXPECT_EQ(nullptr, first.data()); + ExpectPayload(second, 8192, 2); + Message* alias = &second; + second = std::move(*alias); + ExpectPayload(second, 8192, 2); + first = MakeMessage(16, 3); + first.Swap(second); + ExpectPayload(first, 8192, 2); + ExpectPayload(second, 16, 3); + Message empty; + second = std::move(empty); + EXPECT_EQ(nullptr, second.data()); + EXPECT_EQ(nullptr, empty.data()); +} + +TEST(FlatbuffersTest, MergeFromAndCopyFromShareStorage) { + Message source = MakeMessage(128, 7); + Message merged = MakeMessage(16, 1); + merged.MergeFrom(source); + ExpectPayload(merged, 128, 7); + EXPECT_EQ(source.data(), merged.data()); + merged.MergeFrom(merged); + ExpectPayload(merged, 128, 7); + + Message copied = MakeMessage(32, 2); + copied.CopyFrom(source); + ExpectPayload(copied, 128, 7); + EXPECT_EQ(source.data(), copied.data()); + + uint8_t* buffer = static_cast(copied.mutable_data()); + uint8_t* table = buffer + ::flatbuffers::ReadScalar<::flatbuffers::uoffset_t>(buffer); + const ::flatbuffers::soffset_t vtable_offset = + ::flatbuffers::ReadScalar<::flatbuffers::soffset_t>(table); + uint8_t* vtable = table - vtable_offset; + const ::flatbuffers::voffset_t value_offset = + ::flatbuffers::ReadScalar<::flatbuffers::voffset_t>( + vtable + Payload::VT_VALUE); + ASSERT_NE(0, value_offset); + ::flatbuffers::WriteScalar(table + value_offset, 9); + ExpectPayload(source, 128, 9); + ExpectPayload(merged, 128, 9); + ExpectPayload(copied, 128, 9); + + source.Clear(); + ExpectPayload(merged, 128, 9); + ExpectPayload(copied, 128, 9); + + Message empty; + merged.MergeFrom(empty); + EXPECT_EQ(nullptr, merged.data()); + EXPECT_EQ(0u, merged.size()); +} + +TEST(FlatbuffersTest, BuilderMoveAndReuse) { + MessageBuilder first(8); + auto str = first.CreateString(std::string(8192, 'x')); + MessageBuilder second(std::move(first)); + second.Finish(brpc_fbtest::CreatePayload(second, 8, str)); + Message msg = second.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(8192u, msg.GetRoot()->message()->size()); + FinishPayload(first, std::string(16, 'x')); + ExpectPayload(first.ReleaseMessage(), 16); + FinishPayload(second, std::string(63, 'x')); + first = std::move(second); + MessageBuilder* alias = &first; + first = std::move(*alias); + ExpectPayload(first.ReleaseMessage(), 63); + FinishPayload(second, std::string(1024, 'x')); + ExpectPayload(second.ReleaseMessage(), 1024); +} + +TEST(FlatbuffersTest, BuilderSwapFinishedAndUnfinished) { + MessageBuilder first; + FinishPayload(first, std::string(16, 'x')); + MessageBuilder second; + auto str = second.CreateString(std::string(63, 'x')); + first.Swap(second); + ExpectPayload(second.ReleaseMessage(), 16); + first.Finish(brpc_fbtest::CreatePayload(first, 9, str)); + Message msg = first.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(9, msg.GetRoot()->value()); + EXPECT_EQ(63u, msg.GetRoot()->message()->size()); +} + +TEST(FlatbuffersTest, SharedStringsAfterMoveAndImport) { + MessageBuilder first(8); + auto old = first.CreateSharedString("shared"); + MessageBuilder second(std::move(first)); + auto same = second.CreateSharedString("shared"); + auto other = second.CreateSharedString("other"); + std::vector<::flatbuffers::Offset<::flatbuffers::String> > strings = { + old, same, other}; + second.Finish(second.CreateVector(strings)); + Message msg = second.ReleaseMessage(); + auto root = ::flatbuffers::GetRoot< + ::flatbuffers::Vector<::flatbuffers::Offset<::flatbuffers::String> > >( + msg.data()); + EXPECT_EQ("shared", root->Get(0)->str()); + EXPECT_EQ("shared", root->Get(1)->str()); + EXPECT_EQ("other", root->Get(2)->str()); + ::flatbuffers::FlatBufferBuilder foreign; + auto text = foreign.CreateSharedString("shared"); + MessageBuilder imported(std::move(foreign)); + imported.CreateSharedString("after import"); + imported.Finish(brpc_fbtest::CreatePayload(imported, 1, text)); + ASSERT_TRUE(imported.ReleaseMessage().Verify()); +} + +class CountingAllocator : public ::flatbuffers::Allocator { +public: + CountingAllocator(int* allocations, int* frees, int* destructors) + : _allocations(allocations), _frees(frees), _destructors(destructors) {} + ~CountingAllocator() override { ++*_destructors; } + uint8_t* allocate(size_t n) override { + ++*_allocations; + return new uint8_t[n]; + } + void deallocate(uint8_t* p, size_t) override { + ++*_frees; + delete[] p; + } +private: + int* _allocations; + int* _frees; + int* _destructors; +}; + +TEST(FlatbuffersTest, ImportForeignAllocatorAndScratch) { + int allocations = 0; + int frees = 0; + int destructors = 0; + Message msg; + { + ::flatbuffers::FlatBufferBuilder foreign(8, + new CountingAllocator(&allocations, &frees, &destructors), true); + auto text = foreign.CreateString(std::string(8192, 'x')); + const auto table = foreign.StartTable(); + foreign.AddOffset(Payload::VT_MESSAGE, text); + foreign.AddElement(Payload::VT_VALUE, 123, 0); + MessageBuilder imported(std::move(foreign)); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(1, destructors); + imported.Finish(::flatbuffers::Offset(imported.EndTable(table))); + msg = imported.ReleaseMessage(); + FinishPayload(foreign, "reuse source"); + } + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(123, msg.GetRoot()->value()); + EXPECT_EQ(8192u, msg.GetRoot()->message()->size()); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(1, destructors); +} + +TEST(FlatbuffersTest, ImportBorrowedAllocatorDoesNotOwnAllocator) { + int allocations = 0; + int frees = 0; + int destructors = 0; + Message msg; + { + CountingAllocator allocator(&allocations, &frees, &destructors); + ::flatbuffers::FlatBufferBuilder foreign(8, &allocator, false); + FinishPayload(foreign, std::string(1024, 'x')); + MessageBuilder imported(std::move(foreign)); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(0, destructors); + msg = imported.ReleaseMessage(); + } + EXPECT_EQ(1, destructors); + ExpectPayload(msg, 1024); +} + +TEST(FlatbuffersTest, ImportFinishedDefaultBuilderAndEmpty) { + ::flatbuffers::FlatBufferBuilder foreign(8); + FinishPayload(foreign, std::string(8192, 'x')); + MessageBuilder imported(std::move(foreign)); + ExpectPayload(imported.ReleaseMessage(), 8192); + ::flatbuffers::FlatBufferBuilder empty; + imported = std::move(empty); + FinishPayload(imported, std::string(16, 'x')); + ExpectPayload(imported.ReleaseMessage(), 16); + FinishPayload(imported, std::string(63, 'x')); + ::flatbuffers::FlatBufferBuilder& alias = imported; + imported = std::move(alias); + ExpectPayload(imported.ReleaseMessage(), 63); +} + +TEST(FlatbuffersTest, AllocatorFrontBackAndMove) { + SlabAllocator first; + uint8_t* data = first.allocate(64); + EXPECT_EQ(0u, reinterpret_cast(data) % + brpc::flatbuffers::kBufferAlignment); + memset(data, 'f', 16); + memset(data + 48, 'b', 16); + SlabAllocator second(std::move(first)); + size_t previous = 64; + for (size_t next : {128u, 512u, 8192u, 16384u}) { + data = second.reallocate_downward(data, previous, next, 16, 16); + EXPECT_EQ(std::string(16, 'f'), std::string((char*)data, 16)); + EXPECT_EQ(std::string(16, 'b'), std::string((char*)data + next - 16, 16)); + previous = next; + } + first = std::move(second); + SlabAllocator* alias = &first; + first = std::move(*alias); + first.deallocate(data, 16384); + second.deallocate(nullptr, 0); + data = second.allocate(32); + second.deallocate(data, 32); +} + +TEST(FlatbuffersTest, SerializeConstAndRetainWireStorage) { + butil::IOBuf wire; + { + const Message msg = MakeMessage(8192); + ASSERT_TRUE(brpc::flatbuffers::SerializeFbToIOBUF(&msg, wire)); + EXPECT_EQ(msg.size() + msg.get_meta_size(), wire.size()); + } + Message parsed; + const size_t meta = brpc::flatbuffers::kDefaultMetaSize; + ASSERT_TRUE(brpc::flatbuffers::ParseFbFromIOBUF( + &parsed, wire.size() - meta, wire, meta)); + wire.clear(); + ExpectPayload(parsed, 8192); +} + +TEST(FlatbuffersTest, ParseAlignedStorageIsSharedAndRetained) { + Message source = MakeMessage(); + void* raw = nullptr; + ASSERT_EQ(0, posix_memalign(&raw, brpc::flatbuffers::kBufferAlignment, + source.size())); + memcpy(raw, source.data(), source.size()); + int frees = 0; + butil::IOBuf wire; + ASSERT_EQ(0, wire.append_user_data(raw, source.size(), [&frees](void* p) { + ++frees; + free(p); + })); + Message parsed; + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + EXPECT_EQ(raw, parsed.data()); + wire.clear(); + EXPECT_EQ(0, frees); + ExpectPayload(parsed, 16); + parsed.Clear(); + EXPECT_EQ(1, frees); +} + +TEST(FlatbuffersTest, ParseFragmentedUnalignedAndReplace) { + for (size_t length : {16u, 8192u, 32768u}) { + Message source = MakeMessage(length); + butil::IOBuf wire; + const size_t split = source.size() / 2; + void* first = malloc(split); + void* second = malloc(source.size() - split); + memcpy(first, source.data(), split); + memcpy(second, source.data() + split, source.size() - split); + ASSERT_EQ(0, wire.append_user_data(first, split, free)); + ASSERT_EQ(0, wire.append_user_data(second, source.size() - split, free)); + ASSERT_EQ(2u, wire.backing_block_num()); + Message parsed = MakeMessage(1); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + wire.clear(); + ExpectPayload(parsed, length); + EXPECT_EQ(0u, reinterpret_cast(parsed.data()) % + brpc::flatbuffers::kBufferAlignment); + + char* bytes = static_cast(malloc(source.size() + 1)); + bytes[0] = 'h'; + memcpy(bytes + 1, source.data(), source.size()); + ASSERT_EQ(0, wire.append_user_data(bytes, source.size() + 1, free)); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 1)); + EXPECT_NE(bytes + 1, reinterpret_cast(parsed.data())); + wire.clear(); + ExpectPayload(parsed, length); + EXPECT_EQ(1u, parsed.get_meta_size()); + } +} + +TEST(FlatbuffersTest, ParseRejectsBadFramingWithoutChangingMessage) { + Message msg = MakeMessage(16); + const uint8_t* data = msg.data(); + butil::IOBuf buf; + buf.append("abcd"); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 0, 4)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 4, 1)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 5, 0)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 1, 5)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, std::numeric_limits::max(), 5)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 5, std::numeric_limits::max())); + EXPECT_EQ(data, msg.data()); + ExpectPayload(msg, 16); +} + +TEST(FlatbuffersTest, InvalidPayloadAndOptionalString) { + Message msg; + for (size_t size : {1u, 3u, 4u, 16u, 64u}) { + butil::IOBuf buf; + buf.append(std::string(size, '\xff')); + ASSERT_TRUE(msg.parse_msg_from_iobuf(buf, size, 0)); + EXPECT_FALSE(msg.Verify()); + } + MessageBuilder builder; + builder.Finish(brpc_fbtest::CreatePayload(builder)); + msg = builder.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(nullptr, msg.GetRoot()->message()); + // The schema allows an absent string; callers must not dereference it. + msg = MakeMessage(); + memset(msg.mutable_data(), 0xff, sizeof(::flatbuffers::uoffset_t)); + EXPECT_FALSE(msg.Verify()); +} + +TEST(FlatbuffersTest, ShrinkMetadataDoesNotMovePayload) { + Message msg = MakeMessage(16); + const uint8_t* data = msg.data(); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(65)); + ASSERT_NE(nullptr, msg.reduce_meta_size_and_get_buf(36)); + memset(msg.mutable_buf_begin(), 'm', 36); + EXPECT_EQ(data, msg.data()); + ExpectPayload(msg, 16); + EXPECT_EQ(data, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_EQ(data, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(1)); + EXPECT_EQ(0u, msg.get_meta_size()); + butil::IOBuf wire; + ASSERT_TRUE(msg.append_msg_to_iobuf(wire)); + EXPECT_EQ(msg.size(), wire.size()); +} + +void* FailBlockAllocation(size_t) { return nullptr; } + +class FlatbuffersDeathTest : public ::testing::Test { +protected: + void SetUp() override { + _old_style = GTEST_FLAG_GET(death_test_style); + GTEST_FLAG_SET(death_test_style, "threadsafe"); +#if !BRPC_WITH_GLOG + _old = logging::FLAGS_crash_on_fatal_log; + logging::FLAGS_crash_on_fatal_log = false; +#endif + } + void TearDown() override { + GTEST_FLAG_SET(death_test_style, _old_style); +#if !BRPC_WITH_GLOG + logging::FLAGS_crash_on_fatal_log = _old; +#endif + } +private: + bool _old = false; + std::string _old_style; +}; + +TEST_F(FlatbuffersDeathTest, AllocationFailureIsChecked) { + EXPECT_DEATH({ + butil::iobuf::blockmem_allocate = FailBlockAllocation; + SlabAllocator allocator; + allocator.allocate(1024 * 1024); + }, "Fail to allocate"); + EXPECT_DEATH({ + SlabAllocator allocator; + uint8_t* data = allocator.allocate(1024 * 1024); + butil::iobuf::blockmem_allocate = FailBlockAllocation; + allocator.reallocate_downward(data, 1024 * 1024, 2 * 1024 * 1024, 8, 8); + }, "Fail to allocate"); +} + +TEST_F(FlatbuffersDeathTest, RejectsUnsafeAllocatorInputsAndUnfinishedRelease) { + EXPECT_DEATH({ SlabAllocator a; a.allocate(0); }, ""); + EXPECT_DEATH({ SlabAllocator a; a.allocate(std::numeric_limits::max()); }, ""); + EXPECT_DEATH({ + SlabAllocator a; + uint8_t* p = a.allocate(8); + a.reallocate_downward(p, 8, 16, 8, 1); + }, ""); + EXPECT_DEATH({ MessageBuilder builder; builder.ReleaseMessage(); }, "Finish"); +} + +TEST(FlatbuffersDescriptorTest, SparseStableIDsAndCanonicalNames) { + ServiceDescriptor first; + ASSERT_EQ(0, first.init({"test.", "Benchmark", "One Two Three", {2, 5, 1}})); + EXPECT_EQ("test.Benchmark", first.full_name()); + EXPECT_EQ(3, first.method_count()); + EXPECT_EQ(2, first.method(0)->index()); + EXPECT_EQ(5, first.method(1)->index()); + EXPECT_EQ("test.Benchmark.Two", first.method(1)->full_name()); + EXPECT_EQ(&first, first.method(0)->service()); + EXPECT_EQ(first.method(1), first.FindMethodByIndex(5)); + EXPECT_EQ(nullptr, first.FindMethodByIndex(3)); + EXPECT_EQ(nullptr, first.method(-1)); + EXPECT_EQ(nullptr, first.method(3)); + ServiceDescriptor reduced; + ASSERT_EQ(0, reduced.init({"test", "Benchmark", "Three Two", {1, 5}})); + EXPECT_EQ(first.index(), reduced.index()); + EXPECT_EQ(first.FindMethodByIndex(5)->index(), reduced.method(1)->index()); + EXPECT_EQ(first.FindMethodByIndex(5)->full_name(), reduced.method(1)->full_name()); + EXPECT_NE(0, first.init({"test", "Other", "Call", {3}})); + EXPECT_EQ("test.Benchmark", first.full_name()); +} + +TEST(FlatbuffersDescriptorTest, LegacyGlobalAndWhitespace) { + ServiceDescriptor desc; + ASSERT_EQ(0, desc.init({"", "Global", " First\tSecond\n Third ", {}})); + EXPECT_EQ("Global", desc.full_name()); + EXPECT_EQ(3, desc.method_count()); + EXPECT_EQ(0, desc.method(0)->index()); + EXPECT_EQ(2, desc.method(2)->index()); + EXPECT_EQ("Global.First", desc.method(0)->full_name()); +} + +TEST(FlatbuffersDescriptorTest, InvalidTablesAndFailureOwnership) { + const std::vector invalid = { + {"test", "", "A", {}}, {"test", "Service", "", {}}, + {"test", "Service", "A A", {1, 2}}, + {"test", "Service", "A B", {1, 1}}, + {"test", "Service", "A B", {1}}, + {"test", "Service", "A B", {-1, 2}}, + {"test..", "Service", "A", {}}, {".", "Service", "A", {}}, + {"test", "Bad.Name", "A", {}}, {"test", "Service", "1Bad", {}} + }; + ServiceDescriptor desc; + for (const auto& table : invalid) { + EXPECT_NE(0, desc.init(table)); + EXPECT_EQ(0, desc.method_count()); + ServiceDescriptor* output = &desc; + EXPECT_NE(0, brpc::flatbuffers::parse_service_descriptors(table, &output)); + EXPECT_EQ(&desc, output); + } + ASSERT_EQ(0, desc.init({"test", "Service", "A", {0}})); + EXPECT_NE(0, brpc::flatbuffers::parse_service_descriptors( + {"test", "Service", "A", {0}}, nullptr)); + ServiceDescriptor* output = nullptr; + ASSERT_EQ(0, brpc::flatbuffers::parse_service_descriptors( + {"test", "Service", "A", {0}}, &output)); + std::unique_ptr owner(output); + EXPECT_EQ("test.Service", owner->full_name()); +} + +} // namespace +#endif // BRPC_WITH_FLATBUFFERS diff --git a/test/flatbuffers_codegen/CMakeLists.txt b/test/flatbuffers_codegen/CMakeLists.txt new file mode 100644 index 0000000000..cd3436e469 --- /dev/null +++ b/test/flatbuffers_codegen/CMakeLists.txt @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "Expose Protobuf compatibility variables for codegen tests" FORCE) +find_package(Protobuf CONFIG QUIET) +set(CODEGEN_PROTOBUF_CONFIG ${Protobuf_FOUND}) +if(NOT Protobuf_FOUND) + find_package(Protobuf REQUIRED) +endif() +set(CODEGEN_CXX_STANDARD 14) +if(Protobuf_VERSION VERSION_GREATER 4.21) + set(CODEGEN_CXX_STANDARD 17) + if(NOT CODEGEN_PROTOBUF_CONFIG) + message(FATAL_ERROR "Protobuf 5+ needs its CMake config package for codegen tests") + endif() +endif() +message(STATUS "Codegen tests: Protobuf ${Protobuf_VERSION}, C++${CODEGEN_CXX_STANDARD}") +find_program(FLATC_EXECUTABLE flatc) +if(NOT FLATC_EXECUTABLE) + message(FATAL_ERROR "Codegen tests need the official flatc executable") +endif() +execute_process(COMMAND "${FLATC_EXECUTABLE}" --version + OUTPUT_VARIABLE CODEGEN_FLATC_VERSION RESULT_VARIABLE CODEGEN_FLATC_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT CODEGEN_FLATC_RESULT EQUAL 0 OR + NOT CODEGEN_FLATC_VERSION STREQUAL "flatc version 25.2.10") + message(FATAL_ERROR "Codegen tests require official flatc 25.2.10: ${CODEGEN_FLATC_VERSION}") +endif() +find_path(CODEGEN_GFLAGS_INCLUDE_DIR gflags/gflags_declare.h) +if(NOT CODEGEN_GFLAGS_INCLUDE_DIR) + message(FATAL_ERROR "Codegen tests need gflags development headers") +endif() +# execute_process does not inherit a target's compiler flags or macOS SDK. +set(CODEGEN_CXX_FLAGS "${CMAKE_CXX_COMPILER_ARG1} ${CMAKE_CXX_FLAGS}") +foreach(config DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + string(APPEND CODEGEN_CXX_FLAGS + " $<$:${CMAKE_CXX_FLAGS_${config}}>") +endforeach() +if(CMAKE_SYSROOT) + string(APPEND CODEGEN_CXX_FLAGS " --sysroot=\"${CMAKE_SYSROOT}\"") +endif() +if(APPLE) + if(CMAKE_OSX_SYSROOT) + set(CODEGEN_OSX_SYSROOT "${CMAKE_OSX_SYSROOT}") + if(NOT IS_ABSOLUTE "${CODEGEN_OSX_SYSROOT}") + execute_process(COMMAND xcrun --sdk "${CMAKE_OSX_SYSROOT}" --show-sdk-path + OUTPUT_VARIABLE CODEGEN_OSX_SYSROOT RESULT_VARIABLE sdk_result + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT sdk_result EQUAL 0 OR NOT IS_DIRECTORY "${CODEGEN_OSX_SYSROOT}") + message(FATAL_ERROR "Cannot resolve codegen SDK: ${CMAKE_OSX_SYSROOT}") + endif() + endif() + string(APPEND CODEGEN_CXX_FLAGS " -isysroot \"${CODEGEN_OSX_SYSROOT}\"") + endif() + foreach(arch IN LISTS CMAKE_OSX_ARCHITECTURES) + string(APPEND CODEGEN_CXX_FLAGS " -arch ${arch}") + endforeach() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + string(APPEND CODEGEN_CXX_FLAGS + " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() +endif() +message(STATUS "Codegen flatc: ${FLATC_EXECUTABLE} (${CODEGEN_FLATC_VERSION})") +get_filename_component(BRPC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) +set(GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") +file(MAKE_DIRECTORY "${GENERATED_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/include/butil") +# Keep test-only configuration in this build tree, never in the repository. +set(WITH_FLATBUFFERS_VAL 1) +configure_file("${BRPC_SOURCE_DIR}/src/butil/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/butil/config.h") +set(CODEGEN_INCLUDE_DIRS + "${CMAKE_CURRENT_BINARY_DIR}/include" + "${BRPC_SOURCE_DIR}/src" + "${GENERATED_DIR}" + "${FLATBUFFERS_INCLUDE_DIR}" + "${CODEGEN_GFLAGS_INCLUDE_DIR}" + ${Protobuf_INCLUDE_DIRS}) +get_target_property(CODEGEN_PROTOBUF_INCLUDE_DIRS protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) +if(CODEGEN_PROTOBUF_INCLUDE_DIRS) + list(APPEND CODEGEN_INCLUDE_DIRS ${CODEGEN_PROTOBUF_INCLUDE_DIRS}) +endif() +if(TARGET absl::base) + get_target_property(CODEGEN_ABSL_INCLUDE_DIRS absl::base INTERFACE_INCLUDE_DIRECTORIES) + if(CODEGEN_ABSL_INCLUDE_DIRS) + list(APPEND CODEGEN_INCLUDE_DIRS ${CODEGEN_ABSL_INCLUDE_DIRS}) + endif() +endif() +add_custom_command( + OUTPUT "${GENERATED_DIR}/echo.brpc.fb.cpp" + "${GENERATED_DIR}/echo.brpc.fb.h" + "${GENERATED_DIR}/echo_generated.h" + COMMAND "${FLATC_EXECUTABLE}" --cpp -o "${GENERATED_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + COMMAND brpc_flatc -o "${GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + DEPENDS brpc_flatc "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + VERBATIM) +add_library(brpc_codegen_compile OBJECT "${GENERATED_DIR}/echo.brpc.fb.cpp") +target_include_directories(brpc_codegen_compile PRIVATE ${CODEGEN_INCLUDE_DIRS}) +target_compile_features(brpc_codegen_compile PRIVATE cxx_std_${CODEGEN_CXX_STANDARD}) + +add_test(NAME flatbuffers_codegen_acceptance + COMMAND "${CMAKE_COMMAND}" + "-DGENERATOR=$" + "-DFLATC=${FLATC_EXECUTABLE}" + "-DSCHEMA=${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + "-DWORK=${CMAKE_CURRENT_BINARY_DIR}/acceptance" + "-DCXX=${CMAKE_CXX_COMPILER}" + "-DCXX_FLAGS=${CODEGEN_CXX_FLAGS}" + "-DCXX_ID=${CMAKE_CXX_COMPILER_ID}" + "-DIMPLICIT_INCLUDE_DIRS=${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}" + "-DCXX_STANDARD=${CODEGEN_CXX_STANDARD}" + "-DINCLUDE_DIRS=${CODEGEN_INCLUDE_DIRS}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/acceptance.cmake") + +set(BRPC_CODEGEN_BRPC_LIBRARY "" CACHE FILEPATH + "Optional existing FlatBuffers-enabled libbrpc for runtime acceptance") +set(BRPC_CODEGEN_EXTRA_LIBRARIES "" CACHE STRING + "Additional link dependencies of the supplied libbrpc") +if(BRPC_CODEGEN_BRPC_LIBRARY) + find_package(Threads REQUIRED) + find_package(OpenSSL REQUIRED) + find_package(ZLIB REQUIRED) + find_library(CODEGEN_GFLAGS_LIBRARY gflags) + find_library(CODEGEN_LEVELDB_LIBRARY leveldb) + add_executable(brpc_codegen_runtime runtime.cpp $) + target_include_directories(brpc_codegen_runtime PRIVATE ${CODEGEN_INCLUDE_DIRS}) + target_compile_features(brpc_codegen_runtime PRIVATE cxx_std_${CODEGEN_CXX_STANDARD}) + target_link_libraries(brpc_codegen_runtime PRIVATE + "${BRPC_CODEGEN_BRPC_LIBRARY}" protobuf::libprotobuf + ${CODEGEN_GFLAGS_LIBRARY} ${CODEGEN_LEVELDB_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB Threads::Threads + ${CMAKE_DL_LIBS} ${BRPC_CODEGEN_EXTRA_LIBRARIES}) + add_test(NAME flatbuffers_codegen_runtime COMMAND brpc_codegen_runtime) +endif() diff --git a/test/flatbuffers_codegen/acceptance.cmake b/test/flatbuffers_codegen/acceptance.cmake new file mode 100644 index 0000000000..ee8d363699 --- /dev/null +++ b/test/flatbuffers_codegen/acceptance.cmake @@ -0,0 +1,809 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(NOT DEFINED CXX_STANDARD) + set(CXX_STANDARD 14) +endif() +separate_arguments(compiler_flags NATIVE_COMMAND "${CXX_FLAGS}") +message(STATUS "Codegen compiler: ${CXX}; flags: ${CXX_FLAGS}; C++${CXX_STANDARD}") +file(READ "${SCHEMA}" schema_text) +file(MAKE_DIRECTORY "${WORK}") + +function(expect_rejected name replacement) + set(directory "${WORK}/${name}") + file(MAKE_DIRECTORY "${directory}") + string(REPLACE "(id: 7)" "${replacement}" text "${schema_text}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0" OR error STREQUAL "") + message(FATAL_ERROR "${name}: invalid schema was accepted or lacked diagnostic") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "${name}: invalid schema emitted service files") + endif() + message(STATUS "Rejected ${name}: ${error}") +endfunction() + +function(expect_schema_rejected name text diagnostic) + set(directory "${WORK}/${name}") + file(REMOVE_RECURSE "${directory}") + file(MAKE_DIRECTORY "${directory}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${FLATC}" --cpp -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE flatc_result ERROR_VARIABLE flatc_error) + if(NOT "${flatc_result}" STREQUAL "0") + message(FATAL_ERROR "${name}: not a valid official flatc input: ${flatc_error}") + endif() + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + string(FIND "${error}" "${diagnostic}" diagnostic_position) + if("${result}" STREQUAL "0" OR diagnostic_position LESS 0) + message(FATAL_ERROR "${name}: schema was accepted or lacked diagnostic: ${error}") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "${name}: rejected schema emitted service files") + endif() + file(WRITE "${directory}/echo.brpc.fb.h" "previous header\n") + file(WRITE "${directory}/echo.brpc.fb.cpp" "previous source\n") + execute_process(COMMAND "${GENERATOR}" -o "${directory}" + "${directory}/echo.fbs" RESULT_VARIABLE result ERROR_VARIABLE error) + string(FIND "${error}" "${diagnostic}" diagnostic_position) + file(READ "${directory}/echo.brpc.fb.h" old_header) + file(READ "${directory}/echo.brpc.fb.cpp" old_source) + if("${result}" STREQUAL "0" OR diagnostic_position LESS 0 OR + NOT "${old_header}" STREQUAL "previous header\n" OR + NOT "${old_source}" STREQUAL "previous source\n") + message(FATAL_ERROR "${name}: rejection changed existing output files") + endif() + message(STATUS "Rejected ${name} without changing outputs: ${error}") +endfunction() + +function(prepare_output_pair directory) + file(REMOVE_RECURSE "${directory}") + file(MAKE_DIRECTORY "${directory}") + file(WRITE "${directory}/echo.fbs" "${schema_text}") + file(WRITE "${directory}/echo.brpc.fb.h" "previous header\n") + file(WRITE "${directory}/echo.brpc.fb.cpp" "previous source\n") +endfunction() + +function(expect_output_pair_restored name failure) + set(directory "${WORK}/${name}") + prepare_output_pair("${directory}") + execute_process(COMMAND "${CMAKE_COMMAND}" -E env + "BRPC_FLATC_TEST_FAILURE=${failure}" + "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0" OR error STREQUAL "") + message(FATAL_ERROR "${name}: injected failure was not reported") + endif() + file(READ "${directory}/echo.brpc.fb.h" current_header) + file(READ "${directory}/echo.brpc.fb.cpp" current_source) + if(NOT current_header STREQUAL "previous header\n" OR + NOT current_source STREQUAL "previous source\n") + message(FATAL_ERROR "${name}: previous output pair was not restored") + endif() + file(GLOB transaction_files "${directory}/echo.brpc.fb.*.tmp" + "${directory}/echo.brpc.fb.*.bak") + if(transaction_files) + message(FATAL_ERROR "${name}: transaction files remained after rollback") + endif() + message(STATUS "Restored output pair after ${failure}: ${error}") +endfunction() + +function(expect_output_pair_removed name failure remaining_backup) + set(directory "${WORK}/${name}") + prepare_output_pair("${directory}") + execute_process(COMMAND "${CMAKE_COMMAND}" -E env + "BRPC_FLATC_TEST_FAILURE=${failure}" + "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0" OR error STREQUAL "") + message(FATAL_ERROR "${name}: injected restore failure was not reported") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "${name}: partial final output remained") + endif() + if(NOT EXISTS "${directory}/echo.brpc.fb.${remaining_backup}.bak") + message(FATAL_ERROR "${name}: recoverable backup was not retained") + endif() + file(GLOB temporary_outputs "${directory}/echo.brpc.fb.*.tmp") + if(temporary_outputs) + message(FATAL_ERROR "${name}: temporary files remained") + endif() + message(STATUS "Removed final pair after ${failure}: ${error}") +endfunction() + +function(expect_concurrent_output_pair) + set(directory "${WORK}/concurrent_output_pair") + set(output "${directory}/output") + file(REMOVE_RECURSE "${directory}") + file(MAKE_DIRECTORY "${directory}/first" "${directory}/second" "${output}") + set(first_schema + "namespace codegen.concurrent;\ntable Request {}\ntable Response {}\nrpc_service Echo { First(Request):Response (id: 1); }\n") + set(second_schema + "namespace codegen.concurrent;\ntable Request {}\ntable Response {}\nrpc_service Echo { Second(Request):Response (id: 2); }\n") + file(WRITE "${directory}/first/echo.fbs" "${first_schema}") + file(WRITE "${directory}/second/echo.fbs" "${second_schema}") + file(WRITE "${directory}/run.sh" + "#!/bin/sh\n\"${GENERATOR}\" -o \"${output}\" \"${directory}/first/echo.fbs\" &\nfirst=\$!\n\"${GENERATOR}\" -o \"${output}\" \"${directory}/second/echo.fbs\" &\nsecond=\$!\nwait \$first\nfirst_result=\$?\nwait \$second\nsecond_result=\$?\ntest \$first_result -eq 0 -a \$second_result -eq 0\n") + execute_process(COMMAND /bin/sh "${directory}/run.sh" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "concurrent_output_pair: generation failed: ${error}") + endif() + execute_process(COMMAND "${FLATC}" --cpp -o "${output}" + "${directory}/first/echo.fbs" RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "concurrent_output_pair: official flatc failed: ${error}") + endif() + set(includes "-I${output}") + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + execute_process(COMMAND "${CXX}" ${compiler_flags} "-std=c++${CXX_STANDARD}" ${includes} + -c "${output}/echo.brpc.fb.cpp" -o "${directory}/echo.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "concurrent_output_pair: mixed generated files: ${error}") + endif() + file(GLOB transaction_files "${output}/echo.brpc.fb.*.tmp" + "${output}/echo.brpc.fb.*.bak") + if(transaction_files) + message(FATAL_ERROR "concurrent_output_pair: transaction files remained") + endif() + message(STATUS "Serialized concurrent output publication") +endfunction() + +function(expect_rejected_rpc_name name) + set(directory "${WORK}/reserved_${name}") + file(MAKE_DIRECTORY "${directory}") + file(REMOVE "${directory}/echo.brpc.fb.h" "${directory}/echo.brpc.fb.cpp") + string(REPLACE "Inspect(" "${name}(" text "${schema_text}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0") + message(SEND_ERROR "RPC ${name}: generator incorrectly accepted a fixed Stub field name") + return() + endif() + if(NOT error MATCHES "method name collides with generated API: ${name}") + message(FATAL_ERROR "RPC ${name}: expected a name-collision diagnostic: ${error}") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "RPC ${name}: rejected schema emitted service files") + endif() + message(STATUS "Rejected RPC ${name}: ${error}") +endfunction() + +function(expect_compiles name text) + set(standard "${CXX_STANDARD}") + if(ARGC GREATER 2) + set(standard "${ARGV2}") + endif() + set(directory "${WORK}/${name}") + file(MAKE_DIRECTORY "${directory}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${FLATC}" --cpp -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: official flatc failed: ${error}") + endif() + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: brpc_flatc failed: ${error}") + endif() + set(includes "-I${directory}") + set(diagnostics) + foreach(include_dir IN LISTS INCLUDE_DIRS) + if(ARGC GREATER 3) + list(FIND IMPLICIT_INCLUDE_DIRS "${include_dir}" implicit_index) + if(implicit_index GREATER_EQUAL 0) + continue() + endif() + list(APPEND includes -isystem "${include_dir}") + else() + list(APPEND includes "-I${include_dir}") + endif() + endforeach() + if(ARGC GREATER 3) + set(diagnostics ${ARGV3}) + endif() + execute_process(COMMAND "${CXX}" ${compiler_flags} ${diagnostics} "-std=c++${standard}" ${includes} + -c "${directory}/echo.brpc.fb.cpp" -o "${directory}/echo.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: generated code did not compile: ${error}") + endif() + # The generated header must also compile without incidental prior includes. + file(WRITE "${directory}/header.cpp" "#include \"echo.brpc.fb.h\"\n") + execute_process(COMMAND "${CXX}" ${compiler_flags} ${diagnostics} "-std=c++${standard}" ${includes} + -c "${directory}/header.cpp" -o "${directory}/header.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: generated header is not self-contained: ${error}") + endif() + foreach(suffix brpc.fb.h brpc.fb.cpp) + file(READ "${directory}/echo.${suffix}" generated) + if(NOT generated MATCHES "Licensed to the Apache Software Foundation") + message(FATAL_ERROR "${name}: ${suffix} lacks Apache license") + endif() + if(generated MATCHES "FLATBUFFERS_VERSION") + message(FATAL_ERROR "${name}: service output pins FlatBuffers version") + endif() + endforeach() + message(STATUS "Compiled ${name}") +endfunction() + +function(expect_distinct_headers name first_stem second_stem) + set(directory "${WORK}/header_guards_${name}") + set(includes) + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + foreach(side first second) + set(stem "${${side}_stem}") + set(output "${directory}/${side}") + file(MAKE_DIRECTORY "${output}") + string(REPLACE "namespace codegen.example;" "namespace guard_${name}.${side};" + text "${schema_text}") + # Isolate the fixture's global helper table as well as its services. + string(PREPEND text "namespace guard_${name}.${side};\n") + if(side STREQUAL "first") + set(first_text "${text}") + endif() + file(WRITE "${output}/${stem}.fbs" "${text}") + execute_process(COMMAND "${FLATC}" --cpp -o "${output}" "${output}/${stem}.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: official flatc failed: ${error}") + endif() + execute_process(COMMAND "${GENERATOR}" -o "${output}" "${output}/${stem}.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: brpc_flatc failed: ${error}") + endif() + file(WRITE "${output}/single.cpp" + "#include \"${stem}.brpc.fb.h\"\n::guard_${name}::${side}::Echo* service = nullptr;\n") + execute_process(COMMAND "${CXX}" ${compiler_flags} "-std=c++${CXX_STANDARD}" ${includes} + -c "${output}/single.cpp" -o "${output}/single.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: standalone header failed: ${error}") + endif() + endforeach() + foreach(reverse FALSE TRUE) + set(first "#include \"first/${first_stem}.brpc.fb.h\"\n") + set(second "#include \"second/${second_stem}.brpc.fb.h\"\n") + if(reverse) + set(headers "${second}${first}") + else() + set(headers "${first}${second}") + endif() + file(WRITE "${directory}/combined_${reverse}.cpp" + "${headers}::guard_${name}::first::Echo* first_echo = nullptr;\n::guard_${name}::second::Echo* second_echo = nullptr;\n") + execute_process(COMMAND "${CXX}" ${compiler_flags} "-std=c++${CXX_STANDARD}" ${includes} + -c "${directory}/combined_${reverse}.cpp" + -o "${directory}/combined_${reverse}.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: combined headers (reverse=${reverse}) failed: ${error}") + endif() + endforeach() + # A checkout/output directory change must not alter generated identifiers. + set(relocated "${directory}/relocated") + file(MAKE_DIRECTORY "${relocated}") + file(WRITE "${relocated}/${first_stem}.fbs" "${first_text}") + execute_process(COMMAND "${GENERATOR}" -o . "${first_stem}.fbs" + WORKING_DIRECTORY "${relocated}" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: relocated generation failed: ${error}") + endif() + foreach(suffix brpc.fb.h brpc.fb.cpp) + file(READ "${directory}/first/${first_stem}.${suffix}" original) + file(READ "${relocated}/${first_stem}.${suffix}" regenerated) + if(NOT "${original}" STREQUAL "${regenerated}") + message(FATAL_ERROR "${name}: ${suffix} depends on checkout/output paths") + endif() + endforeach() + message(STATUS "Distinct, relocatable header guards: ${name}") +endfunction() + +# Exercise escaped table names as RPC types, not only collision candidates. +foreach(type_name module import synchronized atomic_cancel atomic_commit atomic_noexcept) + set(type_schema "table ${type_name} { value:int; }\ntable Response {}\n") + set(service_schema "rpc_service Echo { Call(${type_name}):Response (id: 1); Reverse(Response):${type_name} (id: 2); }\n") + expect_compiles(escaped_rpc_global_${type_name} + "${type_schema}${service_schema}") + expect_compiles(escaped_rpc_nested_${type_name} + "namespace api.module;\n${type_schema}${service_schema}") + + set(imported_name escaped_rpc_imported_${type_name}) + set(imported_dir "${WORK}/${imported_name}") + file(MAKE_DIRECTORY "${imported_dir}") + file(WRITE "${imported_dir}/types.fbs" "namespace payload;\n${type_schema}") + execute_process(COMMAND "${FLATC}" --cpp -o "${imported_dir}" + "${imported_dir}/types.fbs" RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${imported_name}: official flatc failed (${result}): ${error}") + endif() + expect_compiles(${imported_name} + "include \"types.fbs\"; namespace api; rpc_service Echo { Call(payload.${type_name}):payload.Response (id: 1); Reverse(payload.Response):payload.${type_name} (id: 2); }") +endforeach() +expect_compiles(escaped_rpc_near_name [=[ +namespace api; +table module_ { value:int; } +rpc_service Echo { Call(module_):module_ (id: 1); } +]=]) +expect_compiles(escaped_type_preserves_service_name [=[ +namespace api; +table module { value:int; } +rpc_service module { Call(module):module (id: 1); } +]=]) + +# Exercise the names emitted by official flatc, not only schema type names. +foreach(scope "" "api.") + if(scope STREQUAL "") + set(namespace_decl "") + set(qualified "::") + set(scope_id global) + else() + set(namespace_decl "namespace api;\n") + set(qualified "::api::") + set(scope_id nested) + endif() + set(tables "table Request {}\ntable Response {}\n") + foreach(kind enum_value enum_min enum_any enum_values enum_names enum_name + union_traits union_verify union_vector table_create table_direct) + if(kind MATCHES "^enum_") + set(definitions "enum Collision: byte { Echo_Stub = 0 }\n") + if(kind STREQUAL enum_value) + set(service Collision_Echo) + set(symbol Collision_Echo_Stub) + elseif(kind STREQUAL enum_min) + set(service Collision_MIN) + set(symbol Collision_MIN) + elseif(kind STREQUAL enum_any) + set(definitions "enum Collision: byte (bit_flags) { Flag = 0 }\n") + set(service Collision_ANY) + set(symbol Collision_ANY) + elseif(kind STREQUAL enum_values) + set(service EnumValuesCollision) + set(symbol EnumValuesCollision) + elseif(kind STREQUAL enum_names) + set(service EnumNamesCollision) + set(symbol EnumNamesCollision) + else() + set(service EnumNameCollision) + set(symbol EnumNameCollision) + endif() + elseif(kind MATCHES "^union_") + set(definitions "union Collision { Request }\n") + if(kind STREQUAL union_traits) + set(service CollisionTraits) + elseif(kind STREQUAL union_verify) + set(service VerifyCollision) + else() + set(service VerifyCollisionVector) + endif() + set(symbol "${service}") + elseif(kind STREQUAL table_create) + set(definitions "table Echo_Stub {}\n") + set(service CreateEcho) + set(symbol CreateEcho_Stub) + else() + set(definitions "table Echo { text:string; }\n") + set(service CreateEchoDirect) + set(symbol CreateEchoDirect) + endif() + expect_schema_rejected(${scope_id}_${kind} + "${namespace_decl}${tables}${definitions}rpc_service ${service} { Call(Request):Response (id: 1); }\n" + "generated service class name collides: ${qualified}${symbol}") + endforeach() +endforeach() + +# Pair each root/helper declaration with actual flatc output and legal names. +foreach(symbol GetRoot_Stub GetSizePrefixedRoot_Stub VerifyRoot_StubBuffer + VerifySizePrefixedRoot_StubBuffer FinishRoot_StubBuffer + FinishSizePrefixedRoot_StubBuffer Root_StubIdentifier + Root_StubBufferHasIdentifier SizePrefixedRoot_StubBufferHasIdentifier + Root_StubExtension) + expect_schema_rejected(root_helper_${symbol} + "table Request {} table Response {} table Root_Stub {} root_type Root_Stub; file_identifier \"TEST\"; file_extension \"bin\"; rpc_service ${symbol} { Call(Request):Response (id: 1); }" + "generated service class name collides: ::${symbol}") +endforeach() +file(WRITE "${WORK}/root_types.fbs" + "namespace data; table Request {} table Response {} table Root_Stub {} root_type Root_Stub; file_identifier \"TEST\"; file_extension \"bin\";") +expect_schema_rejected(imported_root_helper + "include \"../root_types.fbs\"; namespace data; rpc_service GetRoot { Call(Request):Response (id: 1); }" + "generated service class name collides: ::data::GetRoot_Stub") +expect_schema_rejected(escaped_table_builder + "table Request {} table Response {} table class {} rpc_service class_Builder { Call(Request):Response (id: 1); }" + "generated service class name collides: ::class_Builder") +expect_compiles(unescaped_table_builder_available + "table Request {} table Response {} table class {} rpc_service classBuilder { Call(Request):Response (id: 1); }") +expect_compiles(escaped_enum_value_available + "table Request {} table Response {} enum E:byte { module = 0 } rpc_service E_module { Call(Request):Response (id: 1); }") +expect_compiles(sparse_enum_no_names_table + "table Request {} table Response {} enum E:int { A = 0, B = 10 } rpc_service EnumNamesE { Call(Request):Response (id: 1); }") +expect_schema_rejected(dense_enum_names_table + "table Request {} table Response {} enum E:int { A = 0, B = 9 } rpc_service EnumNamesE { Call(Request):Response (id: 1); }" + "generated service class name collides: ::EnumNamesE") +expect_compiles(deprecated_string_no_direct + "table Request {} table Response {} table Echo { text:string (deprecated); } rpc_service CreateEchoDirect { Call(Request):Response (id: 1); }") +expect_compiles(union_alias_no_traits + "table Request {} table Response {} union E { A:Request, B:Request } rpc_service ETraits { Call(Request):Response (id: 1); }") +expect_compiles(nonroot_helper_available + "table Request {} table Response {} table Root_Stub {} rpc_service GetRoot { Call(Request):Response (id: 1); }") + +foreach(name __Echo Echo__Name _Echo _echo Echo_ channel_ owned_channel_) + expect_schema_rejected(reserved_service_${name} + "table Request {} table Response {} rpc_service ${name} { Call(Request):Response (id: 1); }" + "reserved C++ identifier") +endforeach() +foreach(name __Call _Call Call__Name) + expect_schema_rejected(reserved_method_${name} + "table Request {} table Response {} rpc_service Echo { ${name}(Request):Response (id: 1); }" + "reserved C++ identifier") +endforeach() +foreach(name __api _Api _api _) + expect_schema_rejected(reserved_namespace_${name} + "namespace ${name}; table Request {} table Response {} rpc_service Echo { Call(Request):Response (id: 1); }" + "reserved C++ identifier") +endforeach() +expect_compiles(global_helper_table [=[ +table BrpcFlatbuffersFail { value:int; } +table Request {} table Response {} +rpc_service Echo { Call(Request):Response (id: 1); } +]=]) +expect_compiles(global_helper_service [=[ +table Request {} table Response {} +rpc_service BrpcFlatbuffersFail { Call(Request):Response (id: 1); } +]=]) +expect_compiles(legal_underscore_names [=[ +namespace api._internal; +table Request {} table Response {} +rpc_service _echo { _call(Request):Response (id: 1); call_(Request):Response (id: 2); } +]=]) +if(CXX_ID MATCHES "Clang") + # Integer-only tables avoid unrelated reserved locals in upstream flatc. + expect_compiles(strict_generated_identifiers + "namespace api; table Request { value:int; } table Response { value:int; } rpc_service Echo { Call(Request):Response (id: 1); }" + "${CXX_STANDARD}" "-Wreserved-identifier;-Werror=reserved-identifier") +else() + message(STATUS "Reserved-name rejection tested; strict diagnostics require Clang") +endif() +expect_compiles(absent_direct_helper [=[ +namespace api; +table Request {} table Response {} table Echo { value:int; } +rpc_service CreateEchoDirect { Call(Request):Response (id: 1); } +]=]) +expect_compiles(cross_namespace_flatc_symbols [=[ +namespace data; +table Request {} table Response {} enum Collision:byte { Echo_Stub = 0 } +namespace api; +rpc_service Collision_Echo { Call(data.Request):data.Response (id: 1); } +]=]) + +foreach(name Stub descriptor GetDescriptor FBCallMethod) + string(CONCAT reserved_service_schema + "table Request {}\ntable Response {}\n" + "rpc_service ${name} { Call(Request):Response (id: 1); }\n") + expect_schema_rejected(service_member_${name} "${reserved_service_schema}" + "service name collides with generated API: ${name}") + expect_schema_rejected(nested_service_member_${name} + "namespace api;\n${reserved_service_schema}" + "service name collides with generated API: ${name}") +endforeach() + +foreach(name Echo Echo_Stub) + foreach(depth "" ".nested") + string(CONCAT namespace_collision_schema + "namespace api.${name}${depth};\n" + "table Request {}\ntable Response {}\n" + "namespace api;\n" + "rpc_service Echo {\n" + " Call(api.${name}${depth}.Request):" + "api.${name}${depth}.Response (id: 1);\n}\n") + expect_schema_rejected(namespace_collision_${name}${depth} + "${namespace_collision_schema}" + "generated service class name collides: ::api::${name}") + endforeach() +endforeach() +file(WRITE "${WORK}/namespace_types.fbs" + "namespace api.Echo; table Request {} table Response {}\n") +expect_schema_rejected(imported_namespace_collision + "include \"../namespace_types.fbs\"; namespace api; + rpc_service Echo { + Call(api.Echo.Request):api.Echo.Response (id: 1); + }" + "generated service class name collides: ::api::Echo") + +expect_distinct_headers(punctuation foo-bar foo_bar) +expect_distinct_headers(letter_case FooBar foobar) +expect_distinct_headers(same_basename echo echo) + +expect_rejected_rpc_name(channel_) +expect_rejected_rpc_name(owned_channel_) +string(REPLACE "rpc_service Echo {" "rpc_service char8_t {" + char8_t_schema "${schema_text}") +expect_schema_rejected(cpp20_char8_t "${char8_t_schema}" + "C++ keyword is not supported: char8_t") +foreach(name brpc butil flatbuffers google std) + set(global_namespace_collision_schema + "table Request {}\ntable Response {}\nrpc_service ${name} { Call(Request):Response (id: 1); }\n") + expect_schema_rejected(global_namespace_${name} + "${global_namespace_collision_schema}" + "generated service class name collides: ::${name}") +endforeach() +set(service_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\nrpc_service Echo { Call(Request):Response (id: 1); }\nrpc_service Echo_Stub { Call(Request):Response (id: 2); }\n") +expect_schema_rejected(service_stub_collision "${service_collision_schema}" + "generated service class name collides: ::codegen::collision::Echo_Stub") +set(stub_service_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\nrpc_service Echo_Stub { Call(Request):Response (id: 2); }\nrpc_service Echo { Call(Request):Response (id: 1); }\n") +expect_schema_rejected(stub_service_collision "${stub_service_collision_schema}" + "generated service class name collides: ::codegen::collision::Echo_Stub") +set(table_service_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\ntable Echo {}\nrpc_service Echo { Call(Request):Response (id: 1); }\n") +expect_schema_rejected(table_service_collision "${table_service_collision_schema}" + "generated service class name collides: ::codegen::collision::Echo") +set(table_stub_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\ntable Echo_Stub {}\nrpc_service Echo { Call(Request):Response (id: 1); }\n") +expect_schema_rejected(table_stub_collision "${table_stub_collision_schema}" + "generated service class name collides: ::codegen::collision::Echo_Stub") +set(table_builder_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\ntable Echo {}\nrpc_service EchoBuilder { Call(Request):Response (id: 1); }\n") +expect_schema_rejected(table_builder_collision "${table_builder_collision_schema}" + "generated service class name collides: ::codegen::collision::EchoBuilder") +set(enum_service_collision_schema + "namespace codegen.collision;\ntable Request {}\ntable Response {}\nenum Echo : byte { Value = 0 }\nrpc_service Echo { Call(Request):Response (id: 1); }\n") +expect_schema_rejected(enum_service_collision "${enum_service_collision_schema}" + "generated service class name collides: ::codegen::collision::Echo") +expect_output_pair_restored(stage_source_failure stage_source) +expect_output_pair_restored(preserve_source_failure preserve_source) +expect_output_pair_restored(publish_header_failure publish_header) +expect_output_pair_restored(publish_source_failure publish_source) +expect_output_pair_removed(restore_header_failure + "publish_source,restore_header" h) +expect_output_pair_removed(restore_source_failure + "publish_source,restore_source" cpp) +expect_concurrent_output_pair() +expect_rejected(missing_id "") +expect_rejected(duplicate_id "(id: 41)") +expect_rejected(negative_id "(id: -1)") +expect_rejected(overflow_id "(id: 2147483648)") +expect_rejected(string_id "(id: \"7\")") +expect_rejected(streaming "(id: 7, streaming: \"server\")") +expect_compiles(sparse_ids "${schema_text}") +file(WRITE "${WORK}/cxx20_probe.cpp" "int main() { return 0; }\n") +execute_process(COMMAND "${CXX}" ${compiler_flags} -std=c++20 -fsyntax-only "${WORK}/cxx20_probe.cpp" + RESULT_VARIABLE cxx20_result ERROR_QUIET) +if("${cxx20_result}" STREQUAL "0") + expect_compiles(cxx20 "${schema_text}" 20) +else() + message(STATUS "Skipping C++20 compile smoke test: compiler lacks -std=c++20") +endif() +set(cross_namespace_schema + "namespace codegen.common;\ntable Request {}\ntable Response {}\nnamespace codegen.first;\nrpc_service Echo { Call(codegen.common.Request):codegen.common.Response (id: 1); }\nnamespace codegen.second;\nrpc_service Echo_Stub { Call(codegen.common.Request):codegen.common.Response (id: 2); }\n") +expect_compiles(cross_namespace_names "${cross_namespace_schema}") +set(cross_namespace_type_schema + "namespace codegen.common;\ntable Request {}\ntable Response {}\ntable Echo {}\nnamespace codegen.api;\nrpc_service Echo { Call(codegen.common.Request):codegen.common.Response (id: 1); }\n") +expect_compiles(cross_namespace_type_names "${cross_namespace_type_schema}") +expect_compiles(namespace_sibling [=[ +namespace api.other.Echo; +table Request {} +table Response {} +namespace api; +rpc_service Echo { + Call(api.other.Echo.Request):api.other.Echo.Response (id: 1); +} +]=]) +expect_compiles(service_member_near_names [=[ +namespace api; +table Request {} +table Response {} +rpc_service channel { Call(Request):Response (id: 1); } +rpc_service channel_handle { Call(Request):Response (id: 1); } +rpc_service owned_channel_handle { Call(Request):Response (id: 1); } +rpc_service StubService { Call(Request):Response (id: 1); } +rpc_service ChannelOwnership { Call(Request):Response (id: 1); } +]=]) + +set(shadow_names request response controller done method std fail BrpcFlatbuffersFail + STUB_OWNS_CHANNEL STUB_DOESNT_OWN_CHANNEL ChannelOwnership) +set(shadow_schema "namespace codegen.names;\ntable Request { text:string; }\ntable Response { text:string; }\nrpc_service Names {\n") +set(method_id 0) +foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND shadow_schema " ${name}(Request):Response (id: ${method_id});\n") +endforeach() +string(APPEND shadow_schema "}\nrpc_service ChannelOwnership { Ping(Request):Response (id: 3); }\n") +expect_compiles(shadowed_names "${shadow_schema}") + +# Supply RUNTIME_LIBRARIES when invoking this script directly to exercise the +# new names against an existing FlatBuffers-enabled bRPC library as well. +if(RUNTIME_LIBRARIES) + set(directory "${WORK}/shadowed_names") + set(runtime_source [=[ +#include "echo.brpc.fb.h" +#include + +using ::brpc::flatbuffers::Message; +using ::google::protobuf::Closure; +using ::google::protobuf::RpcController; + +class Completion : public Closure { +public: + void Run() override { ++runs; } + int runs = 0; +}; + +class Implementation : public ::codegen::names::Names { +public: + int selected = 0; +]=]) + set(method_id 0) + foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND runtime_source + " void ${name}(RpcController*, const Message*, Message*, Closure* completion) override { selected = ${method_id}; completion->Run(); }\n") + endforeach() + string(APPEND runtime_source [=[ +}; + +class LocalChannel : public ::brpc::flatbuffers::RpcChannel { +public: + explicit LocalChannel(::brpc::flatbuffers::Service* service) : service_(service) {} + void FBCallMethod(const ::brpc::flatbuffers::MethodDescriptor* method, + RpcController* controller, const Message* request, + Message* response, Closure* done) override { + service_->FBCallMethod(method, controller, request, response, done); + } +private: + ::brpc::flatbuffers::Service* service_; +}; + +int main() { + ::brpc::flatbuffers::MessageBuilder builder; + builder.Finish(::codegen::names::CreateRequest(builder)); + Message request = builder.ReleaseMessage(); + Message response; + Implementation implementation; + LocalChannel channel(&implementation); + ::codegen::names::Names::Stub stub(&channel); + ::codegen::names::Names defaults; + LocalChannel default_channel(&defaults); + ::codegen::names::Names::Stub default_stub(&default_channel); + ::codegen::names::Names::Stub disconnected(nullptr); + Completion completion; + int expected_runs = 0; +]=]) + set(method_id 0) + foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND runtime_source + " stub.${name}(nullptr, &request, &response, &completion);\n" + " if (implementation.selected != ${method_id} || completion.runs != ++expected_runs) return 1;\n" + " default_stub.${name}(nullptr, &request, &response, &completion);\n" + " if (completion.runs != ++expected_runs) return 2;\n" + " disconnected.${name}(nullptr, &request, &response, &completion);\n" + " if (completion.runs != ++expected_runs) return 3;\n") + endforeach() + string(APPEND runtime_source [=[ + ::codegen::names::ChannelOwnership service; + LocalChannel service_channel(&service); + ::codegen::names::ChannelOwnership::Stub named_stub(&service_channel, + ::brpc::flatbuffers::Service::STUB_DOESNT_OWN_CHANNEL); + named_stub.Ping(nullptr, &request, &response, &completion); + if (completion.runs != ++expected_runs) return 4; + ::codegen::names::Names::Stub owned_stub(new LocalChannel(&implementation), + ::brpc::flatbuffers::Service::STUB_OWNS_CHANNEL); + owned_stub.request(nullptr, &request, &response, &completion); + if (implementation.selected != 7 || completion.runs != ++expected_runs) return 5; + std::cout << "Shadowed-name dispatch, failures, and constructors passed\n"; + return 0; +} +]=]) + file(WRITE "${directory}/runtime.cpp" "${runtime_source}") + set(includes "-I${directory}") + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + execute_process(COMMAND "${CXX}" ${compiler_flags} "-std=c++${CXX_STANDARD}" ${includes} + "${directory}/runtime.cpp" "${directory}/echo.o" ${RUNTIME_LIBRARIES} + -o "${directory}/runtime" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Shadowed-name runtime link failed: ${error}") + endif() + execute_process(COMMAND "${directory}/runtime" + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Shadowed-name runtime failed: ${output}${error}") + endif() + message(STATUS "${output}") +endif() + +string(REPLACE + " Repeat(Request):Response (id: 41);\n Inspect(Request):Response (id: 7);" + " Inspect(Request):Response (id: 7);\n Repeat(Request):Response (id: 41);" + reordered "${schema_text}") +expect_compiles(reordered "${reordered}") +file(READ "${WORK}/reordered/echo.brpc.fb.cpp" reordered_source) +if(NOT reordered_source MATCHES "case 41:" OR NOT reordered_source MATCHES "case 7:") + message(FATAL_ERROR "Reordering changed wire IDs") +endif() +string(FIND "${reordered_source}" "void Echo_Stub::Repeat(" repeat_begin) +if(repeat_begin LESS 0) + message(FATAL_ERROR "Reordered stub lacks Repeat") +endif() +string(SUBSTRING "${reordered_source}" ${repeat_begin} -1 repeat_body) +string(FIND "${repeat_body}" "\n}\n" repeat_end) +string(SUBSTRING "${repeat_body}" 0 ${repeat_end} repeat_body) +if(NOT repeat_body MATCHES "method\\(1\\)") + message(FATAL_ERROR "Reordered stub does not use its dense method position") +endif() + +string(REPLACE "namespace codegen.example;" "" global_schema "${schema_text}") +expect_compiles(global_namespace "${global_schema}") +string(REPLACE "(id: 7)" "(id: 0)" zero_schema "${schema_text}") +expect_compiles(zero_id "${zero_schema}") +string(REPLACE "(id: 7)" "(id: 2147483647)" maximum_schema "${schema_text}") +expect_compiles(maximum_id "${maximum_schema}") + +# Include schemas are read through the official Parser and not re-emitted. +file(MAKE_DIRECTORY "${WORK}/included") +file(WRITE "${WORK}/included/types.fbs" + "namespace shared; table Input { note:string; } table Output { note:string; }\n" + "rpc_service Imported { Ping(Input):Output (id: 3); }\n") +execute_process(COMMAND "${FLATC}" --cpp -o "${WORK}/included" + "${WORK}/included/types.fbs" RESULT_VARIABLE result) +if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Cannot generate included table definitions") +endif() +expect_compiles(included + "include \"types.fbs\"; namespace api; rpc_service Local { Call(shared.Input):shared.Output (id: 17); }") +file(READ "${WORK}/included/echo.brpc.fb.h" included_header) +if(included_header MATCHES "class Imported") + message(FATAL_ERROR "Included service was emitted twice") +endif() + +set(imported_collision_dir "${WORK}/imported_service_collision") +file(REMOVE_RECURSE "${imported_collision_dir}") +file(MAKE_DIRECTORY "${imported_collision_dir}") +file(WRITE "${imported_collision_dir}/types.fbs" + "namespace shared; table Input {} table Output {}\n" + "rpc_service Imported { Ping(Input):Output (id: 3); }\n") +file(WRITE "${imported_collision_dir}/echo.fbs" + "include \"types.fbs\"; namespace shared;\n" + "rpc_service Imported_Stub { Call(Input):Output (id: 7); }\n") +execute_process(COMMAND "${GENERATOR}" -o "${imported_collision_dir}" + "${imported_collision_dir}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) +if("${result}" STREQUAL "0" OR + NOT error MATCHES "generated service class name collides: ::shared::Imported_Stub") + message(FATAL_ERROR + "imported_service_collision: schema was accepted or lacked diagnostic: ${error}") +endif() +if(EXISTS "${imported_collision_dir}/echo.brpc.fb.h" OR + EXISTS "${imported_collision_dir}/echo.brpc.fb.cpp") + message(FATAL_ERROR "imported_service_collision: rejected schema emitted files") +endif() +message(STATUS "Rejected imported service collision: ${error}") diff --git a/test/flatbuffers_codegen/echo.fbs b/test/flatbuffers_codegen/echo.fbs new file mode 100644 index 0000000000..8a59fa8cb9 --- /dev/null +++ b/test/flatbuffers_codegen/echo.fbs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Exercise helper-name collisions in the normal compile and runtime targets. +table BrpcFlatbuffersFail { + marker:int; +} + +namespace codegen.example; + +table Request { + text:string; +} + +table Response { + text:string; +} + +rpc_service Echo { + Repeat(Request):Response (id: 41); + Inspect(Request):Response (id: 7); +} + +rpc_service Other { + Repeat(Request):Response (id: 41); +} + +table module { + value:int; +} + +rpc_service module { + RoundTrip(module):module (id: 17); +} + +root_type Request; diff --git a/test/flatbuffers_codegen/runtime.cpp b/test/flatbuffers_codegen/runtime.cpp new file mode 100644 index 0000000000..133565c536 --- /dev/null +++ b/test/flatbuffers_codegen/runtime.cpp @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "echo.brpc.fb.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +#define CODEGEN_CHECK(condition) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error("check failed: " #condition); \ + } \ + } while (0) + +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::MethodDescriptor; +using brpc::flatbuffers::ServiceDescriptor; +using codegen::example::Echo; +using codegen::example::Other; +using codegen::example::Request; +using codegen::example::Response; +using google::protobuf::Closure; +using google::protobuf::RpcController; + +class Controller : public RpcController { +public: + void Reset() override { failures = 0; error.clear(); } + bool Failed() const override { return failures != 0; } + std::string ErrorText() const override { return error; } + void StartCancel() override {} + void SetFailed(const std::string& reason) override { ++failures; error = reason; } + bool IsCanceled() const override { return false; } + void NotifyOnCancel(Closure*) override {} + int failures = 0; + std::string error; +}; + +class Done : public Closure { +public: + void Run() override { ++runs; } + int runs = 0; +}; + +class DeletingDone : public Closure { +public: + explicit DeletingDone(int* runs) : runs_(runs) {} + void Run() override { ++*runs_; delete this; } +private: + int* runs_; +}; + +Message MakeRequest(const char* text) { + MessageBuilder builder; + const auto value = text ? builder.CreateString(text) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(codegen::example::CreateRequest(builder, value)); + return builder.ReleaseMessage(); +} + +class Implementation : public Echo { +public: + void Repeat(RpcController*, const Message* request, + Message* response, Closure* done) override { + ++calls; + selected = 41; + Reply(request, response, done); + } + void Inspect(RpcController*, const Message* request, + Message* response, Closure* done) override { + ++calls; + selected = 7; + Reply(request, response, done); + } + void Reply(const Message* request, Message* response, Closure* done) { + const auto* input = request->GetRoot(); + MessageBuilder builder; + const auto value = input->text() ? builder.CreateString(input->text()->str()) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(codegen::example::CreateResponse(builder, value)); + *response = builder.ReleaseMessage(); + if (done) { + done->Run(); + } + } + int calls = 0; + int selected = -1; +}; + +class LocalChannel : public brpc::flatbuffers::RpcChannel { +public: + explicit LocalChannel(Echo* service, int* destroyed = nullptr) + : service_(service), destroyed_(destroyed) {} + ~LocalChannel() override { if (destroyed_) { ++*destroyed_; } } + void FBCallMethod(const MethodDescriptor* method, RpcController* controller, + const Message* request, Message* response, Closure* done) override { + last_method = method; + service_->FBCallMethod(method, controller, request, response, done); + } + const MethodDescriptor* last_method = nullptr; +private: + Echo* service_; + int* destroyed_; +}; + +void TestConcurrentDescriptors() { + const int thread_count = 24; + std::atomic start(false); + std::atomic invalid(0); + std::vector descriptors(thread_count); + std::vector threads; + for (int i = 0; i < thread_count; ++i) { + threads.emplace_back([&, i] { + while (!start.load()) { + std::this_thread::yield(); + } + for (int n = 0; n < 1000; ++n) { + descriptors[i] = Echo::descriptor(); + if (descriptors[i]->method_count() != 2 || + descriptors[i]->method(0)->index() != 41 || + descriptors[i]->method(1)->index() != 7) { + ++invalid; + } + } + }); + } + start.store(true); + for (auto& thread : threads) { + thread.join(); + } + CODEGEN_CHECK(invalid.load() == 0); + for (const auto* descriptor : descriptors) { + CODEGEN_CHECK(descriptor == descriptors.front()); + } + const auto* descriptor = descriptors.front(); + CODEGEN_CHECK(descriptor->full_name() == "codegen.example.Echo"); + CODEGEN_CHECK(descriptor->FindMethodByIndex(41) == descriptor->method(0)); + CODEGEN_CHECK(descriptor->FindMethodByIndex(7) == descriptor->method(1)); + CODEGEN_CHECK(descriptor->FindMethodByIndex(0) == nullptr); + CODEGEN_CHECK(descriptor->method(0)->service() == descriptor); +} + +void ExpectFailure(const std::function& call) { + Controller controller; + Done done; + call(&controller, &done); + CODEGEN_CHECK(controller.Failed()); + CODEGEN_CHECK(controller.failures == 1); + CODEGEN_CHECK(!controller.ErrorText().empty()); + CODEGEN_CHECK(done.runs == 1); +} + +void TestDispatchAndErrors() { + Implementation service; + LocalChannel channel(&service); + Echo::Stub stub(&channel); + const auto* descriptor = Echo::descriptor(); + CODEGEN_CHECK(stub.GetDescriptor() == descriptor); + CODEGEN_CHECK(stub.channel() == &channel); + for (const char* text : {static_cast(nullptr), "hello"}) { + Message request = MakeRequest(text); + for (int position = 0; position < 2; ++position) { + Controller controller; + Done done; + Message response; + if (position == 0) { + stub.Repeat(&controller, &request, &response, &done); + } else { + stub.Inspect(&controller, &request, &response, &done); + } + CODEGEN_CHECK(!controller.Failed()); + CODEGEN_CHECK(done.runs == 1); + CODEGEN_CHECK(channel.last_method == descriptor->method(position)); + CODEGEN_CHECK(service.selected == (position == 0 ? 41 : 7)); + CODEGEN_CHECK(response.Verify()); + const auto* output = response.GetRoot(); + CODEGEN_CHECK(text ? output->text() && output->text()->str() == text : + output->text() == nullptr); + } + } + const int calls = service.calls; + Message request = MakeRequest(nullptr); + Message response; + Message invalid; + const MethodDescriptor unknown("Unknown", descriptor, 99); + const MethodDescriptor forged("Forged", descriptor, 41); + for (const auto* method : {static_cast(nullptr), + Other::descriptor()->method(0), &unknown, &forged}) { + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(method, controller, &request, &response, done); + }); + } + for (int position = 0; position < 2; ++position) { + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(position), controller, + &invalid, &response, done); + }); + } + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(0), controller, nullptr, &response, done); + }); + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(0), controller, &request, nullptr, done); + }); + CODEGEN_CHECK(service.calls == calls); + + Echo unimplemented; + for (int position = 0; position < 2; ++position) { + ExpectFailure([&](Controller* controller, Done* done) { + unimplemented.FBCallMethod(descriptor->method(position), controller, + &request, &response, done); + }); + } + ExpectFailure([&](Controller* controller, Done* done) { + unimplemented.Repeat(controller, &request, &response, done); + }); + Echo::Stub disconnected(nullptr); + ExpectFailure([&](Controller* controller, Done* done) { + disconnected.Repeat(controller, &request, &response, done); + }); + ExpectFailure([&](Controller* controller, Done* done) { + disconnected.Inspect(controller, &request, &response, done); + }); + Controller controller; + service.FBCallMethod(nullptr, &controller, &request, &response, nullptr); + CODEGEN_CHECK(controller.failures == 1); + Done done; + service.FBCallMethod(nullptr, nullptr, &request, &response, &done); + CODEGEN_CHECK(done.runs == 1); + int deleted_runs = 0; + service.FBCallMethod(nullptr, nullptr, &request, &response, + new DeletingDone(&deleted_runs)); + CODEGEN_CHECK(deleted_runs == 1); +} + +void TestOwnershipAndAsyncCompletion() { + Implementation service; + int destroyed = 0; + { + Echo::Stub owner(new LocalChannel(&service, &destroyed), + brpc::flatbuffers::Service::STUB_OWNS_CHANNEL); + } + CODEGEN_CHECK(destroyed == 1); + { + LocalChannel borrowed(&service, &destroyed); + { + Echo::Stub stub(&borrowed); + } + CODEGEN_CHECK(destroyed == 1); + } + CODEGEN_CHECK(destroyed == 2); + + class Deferred : public Echo { + public: + void Repeat(RpcController*, const Message*, Message*, Closure* done) override { + pending = done; + } + Closure* pending = nullptr; + } deferred; + Message request = MakeRequest(nullptr); + Message response; + Controller controller; + Done done; + deferred.FBCallMethod(Echo::descriptor()->method(0), &controller, + &request, &response, &done); + CODEGEN_CHECK(!controller.Failed()); + CODEGEN_CHECK(done.runs == 0); + CODEGEN_CHECK(deferred.pending == &done); + deferred.pending->Run(); + CODEGEN_CHECK(done.runs == 1); +} + +void TestEscapedTableTypes() { + class EscapedImplementation : public codegen::example::module { + public: + void RoundTrip(RpcController*, const Message* request, + Message* response, Closure* done) override { + value = request->GetRoot()->value(); + response->CopyFrom(*request); + if (done) { + done->Run(); + } + } + int value = 0; + } service; + MessageBuilder builder; + builder.Finish(codegen::example::Createmodule_(builder, 73)); + Message request = builder.ReleaseMessage(); + Message response; + Controller controller; + Done done; + const auto* descriptor = codegen::example::module::descriptor(); + CODEGEN_CHECK(descriptor->full_name() == "codegen.example.module"); + service.FBCallMethod(descriptor->FindMethodByIndex(17), &controller, + &request, &response, &done); + CODEGEN_CHECK(!controller.Failed()); + CODEGEN_CHECK(service.value == 73); + CODEGEN_CHECK(done.runs == 1); + CODEGEN_CHECK(response.Verify()); + CODEGEN_CHECK(response.GetRoot()->value() == 73); +} + +} // namespace + +int main() { + try { + TestConcurrentDescriptors(); + TestDispatchAndErrors(); + TestOwnershipAndAsyncCompletion(); + TestEscapedTableTypes(); + std::cout << "Descriptor, sparse dispatch, verifier, callback and ownership checks passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/test/flatbuffers_message.fbs b/test/flatbuffers_message.fbs new file mode 100644 index 0000000000..0288499125 --- /dev/null +++ b/test/flatbuffers_message.fbs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace brpc_fbtest; + +table Payload { + value:long; + message:string; + values:[int]; +} + +root_type Payload; diff --git a/tools/flatbuffers/CMakeLists.txt b/tools/flatbuffers/CMakeLists.txt new file mode 100644 index 0000000000..df4f578526 --- /dev/null +++ b/tools/flatbuffers/CMakeLists.txt @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.10) +project(brpc_flatc LANGUAGES CXX) + +find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/idl.h) +find_library(FLATBUFFERS_LIBRARY NAMES flatbuffers) +if(NOT FLATBUFFERS_INCLUDE_DIR OR NOT FLATBUFFERS_LIBRARY) + message(FATAL_ERROR "Install official FlatBuffers development headers and libflatbuffers") +endif() + +function(configure_brpc_flatc target) + target_compile_features(${target} PRIVATE cxx_std_11) + target_include_directories(${target} SYSTEM PRIVATE "${FLATBUFFERS_INCLUDE_DIR}") + target_link_libraries(${target} PRIVATE "${FLATBUFFERS_LIBRARY}") + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${target} PRIVATE -Wall -Wextra -Wpedantic) + endif() +endfunction() + +add_executable(brpc_flatc brpc_flatc.cpp) +configure_brpc_flatc(brpc_flatc) + +include(CTest) +if(BUILD_TESTING) + add_executable(brpc_flatc_test brpc_flatc.cpp) + configure_brpc_flatc(brpc_flatc_test) + target_compile_definitions(brpc_flatc_test PRIVATE BRPC_FLATC_TESTING=1) + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../test/flatbuffers_codegen" + "${CMAKE_CURRENT_BINARY_DIR}/tests") +endif() diff --git a/tools/flatbuffers/README.md b/tools/flatbuffers/README.md new file mode 100644 index 0000000000..93bc4f988d --- /dev/null +++ b/tools/flatbuffers/README.md @@ -0,0 +1,243 @@ +# Standalone FlatBuffers service generator + +`brpc_flatc` generates the bRPC **service abstraction**, not a network protocol or +an adapter for `brpc::Channel`. It uses the installed, official +`flatbuffers/idl.h` Parser and `libflatbuffers`; it does not download or modify a +FlatBuffers fork. The official `flatc --cpp` remains responsible for table types. + +## Build and generate + +The tool needs a C++11 compiler, CMake, and official FlatBuffers **25.2.10** +headers/library. Use the matching `flatc` with default `--cpp` generation; +options that change C++ naming, such as `--scoped-enums`, are not modeled by +this generator. To build only the generator: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=OFF +cmake --build /tmp/brpc-codegen-build -j2 +mkdir -p /tmp/brpc-generated +flatc --cpp -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs +/tmp/brpc-codegen-build/brpc_flatc -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs +``` + +The output directory must already exist. The command line is: + +```text +brpc_flatc [-I include_dir]... [-o existing_output_dir] schema.fbs +``` + +For `echo.fbs`, the two tools produce: + +- Official `flatc`: `echo_generated.h`. +- `brpc_flatc`: `echo.brpc.fb.h` and `echo.brpc.fb.cpp`, both Apache-licensed. + +Compile the generated `.cpp` into the application with C++14 or newer and bRPC +configured with FlatBuffers support. Keep the generated header, official table header, and bRPC +headers on the include path. The generated service files do not pin a +FlatBuffers version; use an official `flatc` matching your installed table +headers, as required by official generated code. + +Schema includes are resolved relative to the input file and through repeated +`-I` options (`-Idir` also works). Run official `flatc --cpp` for each included +schema to obtain its table header. Run `brpc_flatc` separately for each schema +that declares services; imported services are not emitted again. Pass the same +include directories to both tools. This tool processes one `.fbs` per invocation +and follows the official default `*_generated.h` naming convention. + +## Explicit, stable IDs + +```fbs +namespace example.api; + +table Request { text:string; } +table Response { text:string; } + +rpc_service Echo { + Repeat(Request):Response (id: 41); + Inspect(Request):Response (id: 7); +} +``` + +Every RPC method must have an explicit integer `(id: N)` in `[0, 2147483647]`. +IDs must be unique **within a service**. Missing, negative, duplicate, quoted, +or out-of-range IDs fail generation with a diagnostic and nonzero exit status. +Reordering declarations never changes their wire IDs. This does not require +explicit IDs on FlatBuffers table fields. + +The generated names are `example::api::Echo`, +`example::api::Echo_Stub`, and the alias `example::api::Echo::Stub`. +They are **not** the old `EchoStub` spelling. Methods take protobuf +`RpcController`/`Closure` and `brpc::flatbuffers::Message` parameters. Derive from +`Echo` and override the schema methods; supply an implementation of +`brpc::flatbuffers::RpcChannel` to a stub. Channel ownership defaults to borrowed; +`Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. + +This intentionally bounded generator supports unary table RPCs, multiple +services/methods, namespaces, included request/response tables, and absent +optional strings. Streaming and C++ keyword names in service/type/namespace +positions are rejected rather than silently misgenerated. Method names that +collide with generated service APIs are also rejected. Services cannot be named +`Stub`, `descriptor`, `GetDescriptor`, or `FBCallMethod`, because these names +are also members of the generated service class. Generated service and stub +class names must be distinct from other services, schema types, table builder +classes, unscoped enum values, union traits, table creation functions, and +namespace prefixes, including those in imported schemas. C++ reserved +identifiers are rejected in the scopes where they are reserved. A service name +ending in `_` is also rejected because its `_Stub` suffix would produce `__`. +Global service names also cannot shadow +namespaces introduced by the generated header (`brpc`, `butil`, `flatbuffers`, +`google`, or `std`). Schema file basenames may contain ASCII letters, digits, +underscores, dots, and hyphens. The header and source are staged under an +output-directory lock before publication. A failure restores the previous pair +when possible; if rollback also fails, the generator +attempts to remove both final outputs and reports that manual cleanup is needed +when the filesystem refuses. The standalone generator currently requires POSIX +file-locking semantics, including on Linux and macOS. + +## Descriptor and completion contracts + +- `Echo::descriptor()` owns a `ServiceDescriptor` in a function-local static + RAII holder. C++11 initialization is thread-safe; the holder and its owned + method descriptors are destroyed normally at process exit. There is no + leaked singleton allocation or unsynchronized lazy initialization. +- The descriptor table contains the namespace prefix, service name, ordered + method names, and explicit IDs. Stubs use `method(position)`; service dispatch + switches on the stable `method->index()`. +- Service dispatch checks descriptor ownership and identity, non-null request + and response, and `request->Verify()` before calling user code. +- Invalid calls, null stub channels, and default unimplemented methods call + `controller->SetFailed()` when a controller exists, then run non-null `done` + exactly once. Null controllers and callbacks are tolerated on failure. +- Successful dispatch transfers completion responsibility to the user method. + The generated dispatcher does not run `done` again, so asynchronous service + implementations remain possible. User methods and channel implementations + must themselves honor the exactly-once completion contract. + +## Using another FlatBuffers version + +The runtime uses FlatBuffers builder internals that may change between upstream +releases, so the source accepts only 25.2.10. To use another version, build it +into a private prefix, update the version checks, and run the full +verification against that prefix. The steps below use 23.5.26 and were tested +on macOS ARM64 (AppleClang 15) and Ubuntu 22.04 (GCC 11.4, Clang 14). They +cover CMake only; the 23.5.26 Bazel files load `@aspect_rules_js` and `@npm`, +which bRPC does not configure. + +Build FlatBuffers. `CMAKE_INSTALL_LIBDIR=lib` keeps the library where +`verify.sh` looks for it: + +```sh +git clone --depth 1 --branch v23.5.26 https://github.com/google/flatbuffers.git /tmp/flatbuffers-23.5.26 +cmake -S /tmp/flatbuffers-23.5.26 -B /tmp/flatbuffers-23.5.26/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/tmp/flatbuffers-23.5.26-install \ + -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DFLATBUFFERS_BUILD_TESTS=OFF +cmake --build /tmp/flatbuffers-23.5.26/build --target install -j4 +``` + +In the bRPC source tree, update the version checks in +`src/brpc/flatbuffers/message.h`, `tools/flatbuffers/brpc_flatc.cpp` and +`test/flatbuffers_codegen/CMakeLists.txt`: + +```sh +perl -pi -e 's/25\.2\.10/23.5.26/g; + s/(FLATBUFFERS_VERSION_MAJOR != )25\b/${1}23/; + s/(FLATBUFFERS_VERSION_MINOR != )2\b/${1}5/; + s/(FLATBUFFERS_VERSION_REVISION != )10\b/${1}26/' \ + src/brpc/flatbuffers/message.h tools/flatbuffers/brpc_flatc.cpp \ + test/flatbuffers_codegen/CMakeLists.txt +git diff --stat +``` + +The diff should touch 13 lines in these three files. If it does not, the source +has changed since this example was written; review the remaining version +checks by hand. + +Then run the verification with the new prefix: + +```sh +FLATBUFFERS_PREFIX=/tmp/flatbuffers-23.5.26-install \ + bash tools/flatbuffers/verify.sh /tmp/brpc-fb23-verify +``` + +Without `FLATBUFFERS_PREFIX`, the script downloads 25.2.10, which the updated +checks reject. Test your own schemas as well. Other versions may need more than +new version numbers, for example changes for Builder internals or the +generator's naming rules. + +## Pre-submit verification + +Local runs and the Linux/macOS codegen CI jobs use the same entry point: + +```sh +CXX=clang++ bash tools/flatbuffers/verify.sh /tmp/brpc-codegen-verify +``` + +Install CMake, a C++ compiler, protobuf, gflags, LevelDB, OpenSSL, zlib, and +Gperftools first. The root unit-test configuration requires Gperftools even +when only the FlatBuffers target is built. +Use `CMAKE_PREFIX_PATH` for non-system dependencies (on macOS, select the desired +Homebrew protobuf prefix). The script fetches SHA-256-pinned official FlatBuffers +25.2.10 and GTest 1.12.0 archives (URLs and hashes are in `verify.sh`). It enables +`BUILD_UNIT_TESTS`, selects the matching `flatc`, and supplies GTest source through +`BRPC_SYSTEM_GTEST_SOURCE_DIR` with `DOWNLOAD_GTEST=OFF`; no system GTest package +is needed. It builds and runs `brpc_flatbuffers_unittest`, rejecting zero-case, +disabled, or skipped runs, then builds the generator and runs both codegen +acceptance and runtime tests. Missing tests or failures stop the script. +`FLATBUFFERS_PREFIX` may reuse an existing matching installation. +`CODEGEN_RUNTIME=OFF` only builds the generator and runs compile-only acceptance; +it skips GTest, the core bRPC build/unit tests, and runtime tests, even when reusing +a build directory. This is not the full CI check. + +For generator changes, add both rejected and accepted schemas, compile the +actual output of both generators, and demonstrate that a new regression fails +before the fix. Check generated symbol families and global/nested/imported +scopes rather than only the reported spelling. Record compiler, dependency, +SDK, flags, and source hashes; the script prints tool versions, build directories, +archive hashes, verbose CTest results, and the core GTest XML report path. +The same script does not make different runner/compiler versions equivalent. +Ordinary FlatBuffers-OFF jobs are not codegen evidence. + +## Independent acceptance tests + +No root build file needs modification. The default standalone build enables +CTest and additionally needs matching official `flatc`, protobuf, and gflags +development headers: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=ON +cmake --build /tmp/brpc-codegen-build -j2 +ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +``` + +The build compiles the generated service against the real bRPC headers. CTest +rejects missing/duplicate/negative/overflow/string IDs, streaming RPCs, C++20 +keywords, and service/stub names that collide with generated C++ types. +Test-only fault injection covers staging, backup, publication, and rollback +failures; concurrent publishers must also leave a consistent pair. The suite +then compiles the generated service and self-contained header for C++14-or-newer +compatibility, sparse IDs, reordered methods, global/nested +namespaces, zero/max IDs, and included table types. Test configuration headers +and generated artifacts stay in the standalone build directory, not in the +source tree. The name-shadowing cases are compile-only by default; pass +`RUNTIME_LIBRARIES` when invoking `acceptance.cmake` directly to exercise their +runtime checks as well. + +To enable additional behavior tests against an **already built** +FlatBuffers-enabled bRPC library: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build \ + -DBUILD_TESTING=ON \ + -DBRPC_CODEGEN_BRPC_LIBRARY=/path/to/libbrpc.a +cmake --build /tmp/brpc-codegen-build -j2 +ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +``` + +Use `BRPC_CODEGEN_EXTRA_LIBRARIES` for additional dependencies of a custom bRPC +build. The runtime test covers concurrent first descriptor access, sparse +stub/dispatch mapping, absent and present strings, invalid/foreign/forged +methods, failed request verification, null arguments, unimplemented methods, +exactly-once error callbacks (including self-deleting closures), deferred +completion, and channel ownership. It uses a local abstract channel; no network +RPC compatibility is claimed. diff --git a/tools/flatbuffers/brpc_flatc.cpp b/tools/flatbuffers/brpc_flatc.cpp new file mode 100644 index 0000000000..9c4665387f --- /dev/null +++ b/tools/flatbuffers/brpc_flatc.cpp @@ -0,0 +1,894 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if FLATBUFFERS_VERSION_MAJOR != 25 || FLATBUFFERS_VERSION_MINOR != 2 || \ + FLATBUFFERS_VERSION_REVISION != 10 +#error "brpc_flatc requires FlatBuffers 25.2.10 headers and library" +#endif + +namespace { + +const char kLicense[] = R"(// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Generated by brpc_flatc. Do not edit. + +)"; + +const char kArguments[] = + "::google::protobuf::RpcController* controller,\n" + " const ::brpc::flatbuffers::Message* request,\n" + " ::brpc::flatbuffers::Message* response,\n" + " ::google::protobuf::Closure* done"; + +struct Service { + const flatbuffers::ServiceDef* definition; + std::vector ids; +}; + +std::string JoinNamespace(const flatbuffers::Definition& definition, + const std::string& separator) { + std::string result; + if (definition.defined_namespace) { + for (const auto& component : definition.defined_namespace->components) { + if (!result.empty()) { + result += separator; + } + result += component; + } + } + return result; +} + +std::string Qualified(const flatbuffers::Definition& definition) { + const std::string ns = JoinNamespace(definition, "::"); + return "::" + (ns.empty() ? "" : ns + "::") + definition.name; +} + +std::string GuardComponent(const std::string& value) { + static const char digits[] = "0123456789ABCDEF"; + std::string result; + for (unsigned char c : value) { + result += digits[c >> 4]; + result += digits[c & 15]; + } + return result; +} + +bool IsCppIdentifier(const std::string& name) { + static const std::set keywords = { + "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", + "bitor", "bool", "break", "case", "catch", "char", "char8_t", + "char16_t", "char32_t", "class", "compl", "concept", "const", + "const_cast", "consteval", "constexpr", "constinit", "continue", + "co_await", "co_return", "co_yield", "decltype", "default", + "delete", "do", + "double", "dynamic_cast", "else", "enum", "explicit", "export", + "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "long", "mutable", "namespace", "new", + "noexcept", "not", "not_eq", "nullptr", "operator", "or", "or_eq", + "private", "protected", "public", "register", "reinterpret_cast", + "requires", "return", "short", "signed", "sizeof", "static", + "static_assert", "static_cast", "struct", "switch", "template", + "this", "thread_local", "throw", "true", "try", "typedef", + "typeid", "typename", "union", "unsigned", "using", "virtual", + "void", "volatile", "wchar_t", "while", "xor", "xor_eq" + }; + return !name.empty() && keywords.count(name) == 0; +} + +std::string FlatcName(const std::string& name) { + // flatc 25.2.10 escapes a different set than the C++20 rejection list. + static const std::set extra = { + "atomic_cancel", "atomic_commit", "atomic_noexcept", "import", + "module", "synchronized" + }; + const bool newer_keyword = name == "char8_t" || name == "consteval" || + name == "constinit"; + return ((!IsCppIdentifier(name) && !newer_keyword) || extra.count(name)) + ? name + "_" : name; +} + +std::string FlatcNamespace(const flatbuffers::Definition& definition) { + const std::string ns = JoinNamespace(definition, "::"); + return ns.empty() ? "::" : "::" + ns + "::"; +} + +std::string FlatcQualifiedType(const flatbuffers::Definition& definition) { + return FlatcNamespace(definition) + FlatcName(definition.name); +} + +bool IsReservedIdentifier(const std::string& name, bool global) { + return name.find("__") != std::string::npos || + (!name.empty() && name[0] == '_' && + (global || (name.size() > 1 && name[1] >= 'A' && name[1] <= 'Z'))); +} + +bool ValidateName(const flatbuffers::Definition& definition, + std::string* error, bool member = false) { + if (!IsCppIdentifier(definition.name)) { + *error = "C++ keyword is not supported: " + definition.name; + return false; + } + const bool global = !member && (!definition.defined_namespace || + definition.defined_namespace->components.empty()); + if (IsReservedIdentifier(definition.name, global)) { + *error = "reserved C++ identifier: " + definition.name; + return false; + } + if (definition.defined_namespace) { + bool first = true; + for (const auto& component : definition.defined_namespace->components) { + if (IsReservedIdentifier(component, first)) { + *error = "reserved C++ identifier in namespace: " + component; + return false; + } + first = false; + if (!IsCppIdentifier(component)) { + *error = "C++ keyword namespace is not supported: " + component; + return false; + } + } + } + return true; +} + +bool ParseId(const flatbuffers::Value& value, int32_t* id) { + if (value.constant.empty() || + !flatbuffers::IsInteger(value.type.base_type)) { + return false; + } + int64_t number = 0; + for (char c : value.constant) { + if (c < '0' || c > '9') { + return false; + } + number = number * 10 + (c - '0'); + if (number > std::numeric_limits::max()) { + return false; + } + } + *id = static_cast(number); + return true; +} + +void CollectRootNames(const flatbuffers::Parser& parser, + std::set* names) { + if (!parser.root_struct_def_) { + return; + } + const auto& root = *parser.root_struct_def_; + const std::string ns = FlatcNamespace(root); + const std::string name = FlatcName(root.name); + names->insert(ns + "Get" + name); + names->insert(ns + "GetSizePrefixed" + name); + names->insert(ns + "Verify" + name + "Buffer"); + names->insert(ns + "VerifySizePrefixed" + name + "Buffer"); + names->insert(ns + "Finish" + name + "Buffer"); + names->insert(ns + "FinishSizePrefixed" + name + "Buffer"); + if (!parser.file_identifier_.empty()) { + names->insert(ns + name + "Identifier"); + names->insert(ns + name + "BufferHasIdentifier"); + names->insert(ns + "SizePrefixed" + name + "BufferHasIdentifier"); + } + if (!parser.file_extension_.empty()) { + names->insert(ns + name + "Extension"); + } +} + +bool CollectServices(const flatbuffers::Parser& parser, + const std::string& input, const char** include_paths, + std::vector* services, std::string* error) { + std::set generated_class_names = { + "::brpc", "::butil", "::flatbuffers", "::google", "::std" + }; + CollectRootNames(parser, &generated_class_names); + // The parser resets root_type after each include; flatc emits those helpers + // when each imported schema is generated separately. + for (const auto& file : parser.GetIncludedFilesRecursive(input)) { + if (file == input) { + continue; + } + std::string schema; + flatbuffers::Parser included; + if (!flatbuffers::LoadFile(file.c_str(), false, &schema) || + !included.Parse(schema.c_str(), include_paths, file.c_str())) { + *error = "cannot inspect imported root_type: " + file; + return false; + } + CollectRootNames(included, &generated_class_names); + } + // Reserve every namespace prefix, including those from imported schemas. + for (const auto* ns : parser.namespaces_) { + std::string prefix; + for (const auto& component : ns->components) { + prefix += "::" + component; + generated_class_names.insert(prefix); + } + } + for (const auto* definition : parser.services_.vec) { + if (definition->generated) { + const std::string service_name = Qualified(*definition); + generated_class_names.insert(service_name); + generated_class_names.insert(service_name + "_Stub"); + } + } + for (const auto* definition : parser.structs_.vec) { + const std::string ns = FlatcNamespace(*definition); + const std::string name = FlatcName(definition->name); + const std::string type_name = ns + name; + generated_class_names.insert(type_name); + if (!definition->fixed) { + generated_class_names.insert(type_name + "Builder"); + generated_class_names.insert(ns + "Create" + name); + for (const auto* field : definition->fields.vec) { + if (!field->deprecated && + (flatbuffers::IsString(field->value.type) || + flatbuffers::IsVector(field->value.type))) { + generated_class_names.insert( + ns + "Create" + name + "Direct"); + break; + } + } + } + } + // Match the default flatc --cpp API, including non-type declarations. + for (const auto* definition : parser.enums_.vec) { + const std::string ns = FlatcNamespace(*definition); + const std::string name = FlatcName(definition->name); + const std::string enum_name = ns + name; + generated_class_names.insert(enum_name); + for (const auto* value : definition->Vals()) { + generated_class_names.insert(enum_name + "_" + FlatcName(value->name)); + } + if (definition->attributes.Lookup("bit_flags")) { + generated_class_names.insert(enum_name + "_NONE"); + generated_class_names.insert(enum_name + "_ANY"); + } else { + generated_class_names.insert(enum_name + "_MIN"); + generated_class_names.insert(enum_name + "_MAX"); + } + generated_class_names.insert(ns + "EnumValues" + name); + generated_class_names.insert(ns + "EnumName" + name); + if (definition->Distance() / definition->size() < 5) { + generated_class_names.insert(ns + "EnumNames" + name); + } + if (definition->is_union) { + if (!definition->uses_multiple_type_instances) { + generated_class_names.insert(enum_name + "Traits"); + } + generated_class_names.insert(ns + "Verify" + name); + generated_class_names.insert( + ns + "Verify" + name + "Vector"); + } + } + for (const auto* definition : parser.services_.vec) { + // Included schemas are generated separately, just as with flatc --cpp. + if (definition->generated) { + continue; + } + if (!ValidateName(*definition, error)) { + return false; + } + if (definition->name == "Stub" || definition->name == "descriptor" || + definition->name == "GetDescriptor" || + definition->name == "FBCallMethod") { + *error = "service name collides with generated API: " + + definition->name; + return false; + } + const std::string service_name = Qualified(*definition); + const std::string stub_name = service_name + "_Stub"; + if (IsReservedIdentifier(definition->name + "_Stub", false)) { + *error = "generated stub uses reserved C++ identifier: " + stub_name; + return false; + } + if (!generated_class_names.insert(service_name).second) { + *error = "generated service class name collides: " + service_name; + return false; + } + if (!generated_class_names.insert(stub_name).second) { + *error = "generated service class name collides: " + stub_name; + return false; + } + Service service = {definition, {}}; + std::set ids; + if (definition->calls.vec.empty()) { + *error = "rpc_service must contain at least one method: " + + definition->name; + return false; + } + for (const auto* call : definition->calls.vec) { + if (!ValidateName(*call, error, true) || + !ValidateName(*call->request, error) || + !ValidateName(*call->response, error)) { + return false; + } + if (call->name == definition->name || + call->name == definition->name + "_Stub" || + call->name == "descriptor" || call->name == "GetDescriptor" || + call->name == "FBCallMethod" || call->name == "Stub" || + call->name == "channel" || call->name == "channel_" || + call->name == "owned_channel_") { + *error = "method name collides with generated API: " + + call->name; + return false; + } + const auto* attribute = call->attributes.Lookup("id"); + int32_t id = 0; + if (!attribute || !ParseId(*attribute, &id)) { + *error = definition->name + "." + call->name + + " requires an explicit nonnegative int32 (id: N)"; + return false; + } + if (!ids.insert(id).second) { + *error = "duplicate method id " + std::to_string(id) + + " in " + definition->name; + return false; + } + if (call->attributes.Lookup("streaming")) { + *error = "streaming RPC is not supported: " + call->name; + return false; + } + service.ids.push_back(id); + } + services->push_back(service); + } + if (services->empty()) { + *error = "input schema contains no rpc_service to generate"; + return false; + } + return true; +} + +void OpenNamespace(const flatbuffers::Definition& definition, + std::ostream& out) { + if (definition.defined_namespace) { + for (const auto& component : definition.defined_namespace->components) { + out << "namespace " << component << " {\n"; + } + } + out << '\n'; +} + +void CloseNamespace(const flatbuffers::Definition& definition, + std::ostream& out) { + if (definition.defined_namespace) { + const auto& components = definition.defined_namespace->components; + for (auto it = components.rbegin(); it != components.rend(); ++it) { + out << "} // namespace " << *it << '\n'; + } + } + out << '\n'; +} + +void GenerateHeader(const Service& service, std::ostream& out) { + const auto& definition = *service.definition; + const std::string& name = definition.name; + OpenNamespace(definition, out); + out << "class " << name << "_Stub;\n\n" + << "class " << name << " : public ::brpc::flatbuffers::Service {\n" + << "public:\n" + << " typedef " << name << "_Stub Stub;\n" + << " static const ::brpc::flatbuffers::ServiceDescriptor* " + "descriptor();\n" + << " const ::brpc::flatbuffers::ServiceDescriptor* " + "GetDescriptor() override;\n" + << " void FBCallMethod(\n" + << " const ::brpc::flatbuffers::MethodDescriptor* method,\n" + << " " << kArguments << ") override;\n"; + for (const auto* call : definition.calls.vec) { + out << " virtual void " << call->name << "(\n" + << " " << kArguments << ");\n"; + } + out << "};\n\n" + << "class " << name << "_Stub : public " << name << " {\n" + << "public:\n" + << " explicit " << name << "_Stub(\n" + << " ::brpc::flatbuffers::RpcChannel* channel,\n" + << " ::brpc::flatbuffers::Service::ChannelOwnership " + "ownership =\n" + << " ::brpc::flatbuffers::Service::" + "STUB_DOESNT_OWN_CHANNEL);\n" + << " ::brpc::flatbuffers::RpcChannel* channel() const {\n" + << " return channel_;\n" + << " }\n"; + for (const auto* call : definition.calls.vec) { + out << " void " << call->name << "(\n" + << " " << kArguments << ") override;\n"; + } + out << "\nprivate:\n" + << " ::brpc::flatbuffers::RpcChannel* channel_;\n" + << " ::std::unique_ptr<::brpc::flatbuffers::RpcChannel> " + "owned_channel_;\n" + << "};\n\n"; + CloseNamespace(definition, out); +} + +void GenerateFailureHandler(std::ostream& out) { + // A local callable cannot be hidden by schema declarations or RPC names. + out << " const auto fail = [controller, done](const char* reason) {\n" + << " if (controller) {\n" + << " controller->SetFailed(reason);\n" + << " }\n" + << " if (done) {\n" + << " done->Run();\n" + << " }\n" + << " };\n"; +} + +void GenerateSource(const Service& service, std::ostream& out) { + const auto& definition = *service.definition; + const std::string& name = definition.name; + OpenNamespace(definition, out); + out << "const ::brpc::flatbuffers::ServiceDescriptor* " << name + << "::descriptor() {\n" + << " struct Holder {\n" + << " ::brpc::flatbuffers::ServiceDescriptor value;\n" + << " Holder() {\n" + << " const ::brpc::flatbuffers::BrpcDescriptorTable " + "table = {\n" + << " \"" << JoinNamespace(definition, ".") << "\", \"" + << name << "\",\n \""; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + out << (i == 0 ? "" : " ") << definition.calls.vec[i]->name; + } + out << "\", {"; + for (size_t i = 0; i < service.ids.size(); ++i) { + out << (i == 0 ? "" : ", ") << service.ids[i]; + } + out << "}\n };\n" + << " if (value.init(table) != 0) {\n" + << " throw ::std::runtime_error(\n" + << " \"invalid generated service descriptor\");\n" + << " }\n" + << " }\n" + << " };\n" + << " static const Holder holder;\n" + << " return &holder.value;\n" + << "}\n\n" + << "const ::brpc::flatbuffers::ServiceDescriptor* " << name + << "::GetDescriptor() {\n" + << " return descriptor();\n" + << "}\n\n" + << "void " << name << "::FBCallMethod(\n" + << " const ::brpc::flatbuffers::MethodDescriptor* method,\n" + << " " << kArguments << ") {\n"; + GenerateFailureHandler(out); + out << " if (!method || method->service() != descriptor() ||\n" + << " descriptor()->FindMethodByIndex(method->index()) != " + "method) {\n" + << " fail(\"invalid service method\");\n" + << " return;\n" + << " }\n" + << " if (!request || !response) {\n" + << " fail(\"null request or response\");\n" + << " return;\n" + << " }\n" + << " switch (method->index()) {\n"; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + const auto& call = *definition.calls.vec[i]; + out << " case " << service.ids[i] << ":\n" + << " if (!request->Verify<" << FlatcQualifiedType(*call.request) + << ">()) {\n" + << " fail(\"invalid " + << call.request->name << " request\");\n" + << " return;\n" + << " }\n" + << " this->" << call.name + << "(controller, request, response, done);\n" + << " return;\n"; + } + out << " default:\n" + << " fail(\"unknown method id\");\n" + << " return;\n" + << " }\n" + << "}\n\n"; + for (const auto* call : definition.calls.vec) { + out << "void " << name << "::" << call->name << "(\n" + << " " << kArguments << ") {\n" + << " (void)request;\n" + << " (void)response;\n"; + GenerateFailureHandler(out); + out << " fail(\"method not implemented: " + << name << "." << call->name << "\");\n" + << "}\n\n"; + } + out << name << "_Stub::" << name << "_Stub(\n" + << " ::brpc::flatbuffers::RpcChannel* channel,\n" + << " ::brpc::flatbuffers::Service::ChannelOwnership ownership)\n" + << " : channel_(channel),\n" + << " owned_channel_(\n" + << " ownership == " + "::brpc::flatbuffers::Service::STUB_OWNS_CHANNEL\n" + << " ? channel : nullptr) {}\n\n"; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + out << "void " << name << "_Stub::" << definition.calls.vec[i]->name + << "(\n " << kArguments << ") {\n"; + GenerateFailureHandler(out); + out << " if (!channel_) {\n" + << " fail(\"null RPC channel\");\n" + << " return;\n" + << " }\n" + << " channel_->FBCallMethod(\n" + << " descriptor()->method(" << i + << "), controller, request, response, done);\n" + << "}\n\n"; + } + CloseNamespace(definition, out); +} + +bool TestFailureEnabled(const char* step) { +#if defined(BRPC_FLATC_TESTING) + const char* value = std::getenv("BRPC_FLATC_TEST_FAILURE"); + if (value) { + const std::string failures = "," + std::string(value) + ","; + return failures.find("," + std::string(step) + ",") != + std::string::npos; + } +#else + (void)step; +#endif + return false; +} + +bool WriteFile(const std::string& path, const std::string& contents, + const char* step) { + if (TestFailureEnabled(step)) { + errno = EIO; + return false; + } + std::ofstream stream(path.c_str(), std::ios::binary | std::ios::trunc); + stream << contents; + stream.close(); + return !stream.fail(); +} + +bool RenameFile(const std::string& from, const std::string& to, + const char* step) { + if (TestFailureEnabled(step)) { + errno = EIO; + return false; + } + return std::rename(from.c_str(), to.c_str()) == 0; +} + +class OutputDirectoryLock { +public: + explicit OutputDirectoryLock(const std::string& directory) + : fd_(open(directory.c_str(), O_RDONLY)) { + if (fd_ < 0) { + return; + } + while (flock(fd_, LOCK_EX) != 0) { + if (errno != EINTR) { + close(fd_); + fd_ = -1; + return; + } + } + } + + ~OutputDirectoryLock() { + if (fd_ >= 0) { + close(fd_); + } + } + + bool acquired() const { return fd_ >= 0; } + +private: + OutputDirectoryLock(const OutputDirectoryLock&) = delete; + OutputDirectoryLock& operator=(const OutputDirectoryLock&) = delete; + + int fd_; +}; + +bool RemoveFile(const std::string& path) { + return std::remove(path.c_str()) == 0 || errno == ENOENT; +} + +bool IsRegularFileOrMissing(const std::string& path) { + struct stat file_status; + if (lstat(path.c_str(), &file_status) == 0) { + return S_ISREG(file_status.st_mode); + } + return errno == ENOENT; +} + +bool MoveExistingFile(const std::string& path, const std::string& backup, + const char* step, bool* moved) { + *moved = false; + if (RenameFile(path, backup, step)) { + *moved = true; + return true; + } + return errno == ENOENT; +} + +bool RestoreFile(const std::string& path, const std::string& backup, + const char* step, bool had_original) { + if (!RemoveFile(path)) { + return false; + } + return !had_original || RenameFile(backup, path, step); +} + +bool RemoveOutputPair(const std::string& header_path, + const std::string& source_path) { + const bool header_removed = RemoveFile(header_path); + const bool source_removed = RemoveFile(source_path); + return header_removed && source_removed; +} + +bool PublishFiles(const std::string& output_dir, + const std::string& header_path, + const std::string& header_contents, + const std::string& source_path, + const std::string& source_contents, + std::string* error) { + OutputDirectoryLock lock(output_dir); + if (!lock.acquired()) { + *error = "cannot lock output directory"; + return false; + } + const std::string header_tmp = header_path + ".tmp"; + const std::string source_tmp = source_path + ".tmp"; + const std::string header_backup = header_path + ".bak"; + const std::string source_backup = source_path + ".bak"; + if (!RemoveFile(header_tmp) || + !WriteFile(header_tmp, header_contents, "stage_header")) { + RemoveFile(header_tmp); + *error = "cannot stage header output"; + return false; + } + if (!RemoveFile(source_tmp) || + !WriteFile(source_tmp, source_contents, "stage_source")) { + RemoveFile(header_tmp); + RemoveFile(source_tmp); + *error = "cannot stage source output"; + return false; + } + if (!IsRegularFileOrMissing(header_path) || + !IsRegularFileOrMissing(source_path) || + !RemoveFile(header_backup) || !RemoveFile(source_backup)) { + RemoveFile(header_tmp); + RemoveFile(source_tmp); + *error = "output or backup path is not replaceable"; + return false; + } + + bool had_header = false; + bool had_source = false; + if (!MoveExistingFile(header_path, header_backup, + "preserve_header", &had_header)) { + RemoveFile(header_tmp); + RemoveFile(source_tmp); + *error = "cannot preserve previous header"; + return false; + } + if (!MoveExistingFile(source_path, source_backup, + "preserve_source", &had_source)) { + const bool restored = RestoreFile( + header_path, header_backup, "restore_header", had_header); + RemoveFile(header_tmp); + RemoveFile(source_tmp); + if (!restored) { + const bool cleared = RemoveOutputPair(header_path, source_path); + *error = cleared + ? "cannot preserve source or restore header" + : "cannot preserve source; manual cleanup required"; + } else { + *error = "cannot preserve previous source"; + } + return false; + } + + if (!RenameFile(header_tmp, header_path, "publish_header") || + !RenameFile(source_tmp, source_path, "publish_source")) { + const bool header_restored = RestoreFile( + header_path, header_backup, "restore_header", had_header); + const bool source_restored = RestoreFile( + source_path, source_backup, "restore_source", had_source); + RemoveFile(header_tmp); + RemoveFile(source_tmp); + if (!header_restored || !source_restored) { + const bool cleared = RemoveOutputPair(header_path, source_path); + *error = cleared + ? "cannot publish or restore output files" + : "cannot publish output files; manual cleanup required"; + } else { + *error = "cannot publish output files"; + } + return false; + } + + RemoveFile(header_backup); + RemoveFile(source_backup); + return true; +} + +void Usage(std::ostream& out) { + out << "Usage: brpc_flatc [-I include_dir]... " + "[-o existing_output_dir] schema.fbs\n" + << "Run official flatc --cpp separately to produce " + "schema_generated.h.\n"; +} + +int Run(int argc, char** argv) { + if (std::string(flatbuffers::FLATBUFFERS_VERSION()) != "25.2.10") { + std::cerr << "brpc_flatc: requires FlatBuffers 25.2.10 library\n"; + return 1; + } + if (argc == 2 && std::string(argv[1]) == "--version") { + std::cout << "brpc_flatc (FlatBuffers 25.2.10)\n"; + return 0; + } + std::string input; + std::string output_dir = "."; + std::vector include_dirs; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--help" || arg == "-h") { + Usage(std::cout); + return 0; + } + if (arg == "-o" || arg == "-I") { + if (++i == argc) { + Usage(std::cerr); + return 1; + } + if (arg == "-o") { + output_dir = argv[i]; + } else { + include_dirs.push_back(argv[i]); + } + } else if (arg.compare(0, 2, "-I") == 0 && arg.size() > 2) { + include_dirs.push_back(arg.substr(2)); + } else if (arg.empty() || arg[0] == '-' || !input.empty()) { + Usage(std::cerr); + return 1; + } else { + input = arg; + } + } + if (input.size() < 5 || input.substr(input.size() - 4) != ".fbs" || + output_dir.empty()) { + Usage(std::cerr); + return 1; + } + const size_t slash = input.find_last_of("/\\"); + const std::string basename = input.substr( + slash == std::string::npos ? 0 : slash + 1); + const std::string stem = basename.substr(0, basename.size() - 4); + if (stem.empty() || stem.find_first_not_of( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-") != + std::string::npos) { + std::cerr << "brpc_flatc: unsupported schema filename\n"; + return 1; + } + include_dirs.push_back(slash == std::string::npos ? "." : + input.substr(0, slash == 0 ? 1 : slash)); + std::vector include_paths; + for (const auto& directory : include_dirs) { + include_paths.push_back(directory.c_str()); + } + include_paths.push_back(nullptr); + std::ifstream stream(input.c_str(), std::ios::binary); + if (!stream) { + std::cerr << "brpc_flatc: cannot read " << input << '\n'; + return 1; + } + const std::string schema((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + if (stream.bad() || schema.find('\0') != std::string::npos) { + std::cerr << "brpc_flatc: invalid schema input\n"; + return 1; + } + flatbuffers::Parser parser; + if (!parser.Parse(schema.c_str(), include_paths.data(), input.c_str())) { + std::cerr << parser.error_ << '\n'; + return 1; + } + std::vector services; + std::string error; + if (!CollectServices(parser, input, include_paths.data(), &services, &error)) { + std::cerr << "brpc_flatc: " << error << '\n'; + return 1; + } + // Coexisting service headers cannot define the same qualified service. + // Preserve filename bytes and that service identity, without tying output + // to checkout paths or folding punctuation and case into the same guard. + const std::string guard = + "BRPC_FLATBUFFERS_GENERATED_" + GuardComponent(stem) + "_" + + GuardComponent(Qualified(*services.front().definition)) + "_H_"; + std::ostringstream header; + header << kLicense << "#ifndef " << guard << "\n#define " << guard << "\n\n" + << "#include \n" + << "#include \n" + << "#include \"brpc/flatbuffers/message.h\"\n" + << "#include \"brpc/flatbuffers/service.h\"\n" + << "#include \"" << stem << "_generated.h\"\n\n" + << "#if !BRPC_WITH_FLATBUFFERS\n" + << "#error \"Generated services require BRPC_WITH_FLATBUFFERS\"\n" + << "#endif\n\n"; + std::ostringstream source; + source << kLicense << "#include \"" << stem << ".brpc.fb.h\"\n\n" + << "#include \n\n"; + for (const auto& service : services) { + GenerateHeader(service, header); + GenerateSource(service, source); + } + header << "#endif // " << guard << '\n'; + if (output_dir.back() != '/') { + output_dir += '/'; + } + if (!PublishFiles(output_dir, + output_dir + stem + ".brpc.fb.h", header.str(), + output_dir + stem + ".brpc.fb.cpp", source.str(), + &error)) { + std::cerr << "brpc_flatc: cannot write output in " << output_dir + << ": " << error << '\n'; + return 1; + } + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + try { + return Run(argc, argv); + } catch (const std::exception& error) { + std::cerr << "brpc_flatc: " << error.what() << '\n'; + return 1; + } +} diff --git a/tools/flatbuffers/verify.sh b/tools/flatbuffers/verify.sh new file mode 100644 index 0000000000..a0ce7ae7f4 --- /dev/null +++ b/tools/flatbuffers/verify.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +if [[ $# != 1 ]]; then + printf 'Usage: bash tools/flatbuffers/verify.sh \n' >&2 + exit 2 +fi +root=$(cd "$(dirname "$0")/../.." && pwd) +mkdir -p "$1" +work=$(cd "$1" && pwd) +cmake=${CMAKE:-cmake} +ctest=${CTEST:-ctest} +cxx=${CXX:-c++} +jobs=${JOBS:-2} +runtime=${CODEGEN_RUNTIME:-ON} +case "$runtime" in + ON|OFF) ;; + *) printf 'CODEGEN_RUNTIME must be ON or OFF\n' >&2; exit 2 ;; +esac + +printf 'Source: %s\nWork: %s\nCODEGEN_RUNTIME: %s\n' "$root" "$work" "$runtime" +uname -sm +"$cxx" --version +"$cmake" --version +"$ctest" --version +"$cmake" -E sha256sum "$root/tools/flatbuffers/brpc_flatc.cpp" \ + "$root/test/flatbuffers_codegen/acceptance.cmake" \ + "$root/test/flatbuffers_codegen/CMakeLists.txt" \ + "$root/test/flatbuffers_codegen/echo.fbs" \ + "$root/test/flatbuffers_codegen/runtime.cpp" + +# Keep the archive and checksum in sync with MODULE.bazel and WORKSPACE. +if [[ -z ${FLATBUFFERS_PREFIX:-} ]]; then + archive="$work/flatbuffers-25.2.10.tar.gz" + curl --fail --location --retry 3 \ + https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz \ + --output "$archive" + checksum=$("$cmake" -E sha256sum "$archive") + printf '%s\n' "$checksum" + if [[ ${checksum%% *} != b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421 ]]; then + printf 'FlatBuffers archive checksum mismatch\n' >&2 + exit 1 + fi + tar -xzf "$archive" -C "$work" + fb="$work/flatbuffers-install" + printf 'FlatBuffers build: %s\n' "$work/flatbuffers-build" + "$cmake" -S "$work/flatbuffers-25.2.10" -B "$work/flatbuffers-build" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER="$cxx" \ + -DCMAKE_INSTALL_PREFIX="$fb" -DFLATBUFFERS_BUILD_TESTS=OFF \ + -DFLATBUFFERS_BUILD_FLATC=ON + "$cmake" --build "$work/flatbuffers-build" --target install --parallel "$jobs" +else + fb=$FLATBUFFERS_PREFIX +fi +printf 'FlatBuffers prefix: %s\n' "$fb" +"$fb/bin/flatc" --version + +runtime_library= +codegen_tests=(flatbuffers_codegen_acceptance) +if [[ $runtime == ON ]]; then + # Match the root GTest version without its unverified Git download path. + archive="$work/googletest-release-1.12.0.tar.gz" + curl --fail --location --retry 3 \ + https://github.com/google/googletest/archive/refs/tags/release-1.12.0.tar.gz \ + --output "$archive" + checksum=$("$cmake" -E sha256sum "$archive") + printf '%s\n' "$checksum" + if [[ ${checksum%% *} != 2a4f11dce6188b256f3650061525d0fe352069e5c162452818efbbf8d0b5fe1c ]]; then + printf 'GTest archive checksum mismatch\n' >&2 + exit 1 + fi + tar -xzf "$archive" -C "$work" + gtest="$work/googletest-release-1.12.0" + printf 'GTest source (1.12.0): %s\nbRPC build: %s\n' "$gtest" "$work/brpc" + "$cmake" -S "$root" -B "$work/brpc" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_CXX_COMPILER="$cxx" -DBUILD_SHARED_LIBS=ON \ + -DBUILD_UNIT_TESTS=ON -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR="$gtest" \ + -DWITH_FLATBUFFERS=ON -DFLATBUFFERS_INCLUDE_DIR="$fb/include" \ + -DFLATBUFFERS_FLATC_EXECUTABLE="$fb/bin/flatc" \ + -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR:-}" \ + -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}" + "$cmake" --build "$work/brpc" --target brpc_flatbuffers_unittest --parallel "$jobs" + # A registered CTest can still pass with zero GTest cases or all cases skipped. + report=$(mktemp "$work/brpc/flatbuffers-gtest.XXXXXX") + printf 'Core GTest XML: %s\n' "$report" + ( + unset GTEST_TOTAL_SHARDS GTEST_SHARD_INDEX GTEST_SHARD_STATUS_FILE + GTEST_FILTER='*' GTEST_REPEAT=1 GTEST_OUTPUT="xml:$report" \ + "$ctest" --test-dir "$work/brpc" --no-tests=error \ + -R '^brpc_flatbuffers_unittest$' --output-on-failure --verbose + ) + if ! grep -Eq '&2 + exit 1 + fi + "$cmake" --build "$work/brpc" --target brpc-shared --parallel "$jobs" + case $(uname -s) in + Darwin) extension=dylib ;; + *) extension=so ;; + esac + runtime_library="$work/brpc/output/lib/libbrpc.$extension" + codegen_tests+=(flatbuffers_codegen_runtime) +else + printf 'Compile-only codegen verification: skipping core and runtime tests\n' +fi +printf 'Codegen build: %s\n' "$work/codegen" +"$cmake" -S "$root/tools/flatbuffers" -B "$work/codegen" \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON \ + -DCMAKE_CXX_COMPILER="$cxx" -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}" \ + -DFLATBUFFERS_INCLUDE_DIR="$fb/include" \ + -DFLATBUFFERS_LIBRARY="$fb/lib/libflatbuffers.a" \ + -DFLATC_EXECUTABLE="$fb/bin/flatc" -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR:-}" \ + -DBRPC_CODEGEN_BRPC_LIBRARY="$runtime_library" +"$cmake" --build "$work/codegen" --parallel "$jobs" +"$work/codegen/brpc_flatc" --version +for test_name in "${codegen_tests[@]}"; do + "$ctest" --test-dir "$work/codegen" --no-tests=error \ + -R "^${test_name}$" --output-on-failure --verbose +done