From bf213c183ac76bdc5d3456aaa7619cb2a6b2ff15 Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Thu, 25 Jun 2026 19:40:59 +0900 Subject: [PATCH 1/3] dockerfile: install libzstd-dev for kafka zstd compression The container builder stages did not install libzstd-dev, so librdkafka's find_package(ZSTD) failed and WITH_ZSTD was disabled. Kafka producers in the resulting images could not use zstd compression. Add libzstd-dev to the builder and debug builder stages so librdkafka is compiled with zstd support. Signed-off-by: Domenic Bottini --- dockerfiles/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 919b642cc3f..d6505a4d2c8 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -49,6 +49,7 @@ RUN apt-get update && \ pkg-config \ libsystemd-dev \ zlib1g-dev \ + libzstd-dev \ libpq-dev \ postgresql-server-dev-all \ flex \ @@ -262,7 +263,7 @@ RUN apt-get update && \ openssl \ htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \ make tar flex bison \ - libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \ + libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libzstd-dev libpq-dev libyaml-dev postgresql-server-dev-all \ && apt-get satisfy -y cmake "cmake (<< 4.0)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From cbca3a907213558bbf1dcae1336ba23b3411278c Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Thu, 25 Jun 2026 19:41:39 +0900 Subject: [PATCH 2/3] tests: runtime: cover librdkafka zstd compression support Add a runtime test that sets compression.codec=zstd on a librdkafka conf. rd_kafka_conf_set only accepts the zstd codec when librdkafka was built with WITH_ZSTD, so this guards against images shipping without zstd support. Signed-off-by: Domenic Bottini --- tests/runtime/out_kafka.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/runtime/out_kafka.c b/tests/runtime/out_kafka.c index 4c5c852bc44..d18ff7f5643 100644 --- a/tests/runtime/out_kafka.c +++ b/tests/runtime/out_kafka.c @@ -3,9 +3,34 @@ #include #include "flb_tests_runtime.h" +#include "rdkafka.h" + /* Test data */ #include "data/td/json_td.h" +/* + * Ensure librdkafka was compiled with zstd support so Kafka producers can + * negotiate zstd compression. Setting compression.codec to zstd only succeeds + * when WITH_ZSTD was enabled at build time, otherwise rd_kafka_conf_set + * reports the codec as not built in. + */ +void flb_test_zstd_compression_available() +{ + rd_kafka_conf_t *conf; + rd_kafka_conf_res_t res; + char errstr[512] = {0}; + + conf = rd_kafka_conf_new(); + TEST_CHECK(conf != NULL); + + res = rd_kafka_conf_set(conf, "compression.codec", "zstd", + errstr, sizeof(errstr)); + TEST_CHECK(res == RD_KAFKA_CONF_OK); + TEST_MSG("compression.codec=zstd rejected: %s", errstr); + + rd_kafka_conf_destroy(conf); +} + void flb_test_raw_format() { @@ -44,6 +69,7 @@ void flb_test_raw_format() } TEST_LIST = { + { "zstd_compression_available", flb_test_zstd_compression_available }, { "raw_format", flb_test_raw_format }, { NULL, NULL }, }; From bf2c96ca8490a3a13a7a2ba875201c94da73822e Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Mon, 29 Jun 2026 16:45:44 +0900 Subject: [PATCH 3/3] tests: runtime: add librdkafka include path for out_kafka The out_kafka runtime test includes librdkafka's public rdkafka.h header directly, but the generic runtime test build loop does not add the kafka include path the in_kafka/out_kafka plugins rely on. This breaks the build with 'rdkafka.h: No such file or directory'. Add the librdkafka include directory to the flb-rt-out_kafka target, covering both the system librdkafka (pkg-config) and bundled librdkafka builds. Co-authored-by: Rovo Dev (Claude) Signed-off-by: Domenic Bottini --- tests/runtime/CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 645d159fec8..fd14785763a 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -306,3 +306,14 @@ foreach(source_file ${CHECK_PROGRAMS}) set_property(TARGET ${source_file_we} APPEND_STRING PROPERTY COMPILE_FLAGS "-D${o_source_file_we}") endif() endforeach() + +# The out_kafka runtime test includes librdkafka's public header directly, so it +# needs the same include path the in_kafka/out_kafka plugins use. Cover both the +# system librdkafka (pkg-config) and the bundled librdkafka build. +if(TARGET flb-rt-out_kafka) + if(DEFINED KAFKA_INCLUDEDIR) + target_include_directories(flb-rt-out_kafka PRIVATE ${KAFKA_INCLUDEDIR}/librdkafka) + endif() + target_include_directories(flb-rt-out_kafka PRIVATE + ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src) +endif()