From f9684209738ac17f89994ab77751c2cae9314c3f Mon Sep 17 00:00:00 2001 From: Rafael Vanoni Date: Tue, 3 Feb 2026 14:59:12 -0800 Subject: [PATCH 01/18] Fix aliasing violations in nmsg --- nmsg/base/dnsqr.c | 4 +++- nmsg/input_json.c | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nmsg/base/dnsqr.c b/nmsg/base/dnsqr.c index 103754c7..d75bcc2d 100644 --- a/nmsg/base/dnsqr.c +++ b/nmsg/base/dnsqr.c @@ -773,6 +773,7 @@ get_af(const char *addr) { static char * addrs_to_bpf(const char *addrs, const char *bpfdir, int af) { char *ret, *tok_addrs, *addr, *saveptr; + uint8_t *ubuf_str; size_t retsz; int addr_af; ubuf *bpf; @@ -815,8 +816,9 @@ addrs_to_bpf(const char *addrs, const char *bpfdir, int af) { free(tok_addrs); ubuf_cterm(bpf); - ubuf_detach(bpf, (uint8_t **) &ret, &retsz); + ubuf_detach(bpf, &ubuf_str, &retsz); ubuf_destroy(&bpf); + ret = (char *)ubuf_str; return (ret); } diff --git a/nmsg/input_json.c b/nmsg/input_json.c index 2c377c06..0a5a8497 100644 --- a/nmsg/input_json.c +++ b/nmsg/input_json.c @@ -24,20 +24,23 @@ #if (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) nmsg_res _input_kafka_json_read(nmsg_input_t input, nmsg_message_t *msg) { + uint8_t *ubuf_str; nmsg_res res; char *buf; size_t buf_len; - res = kafka_read_start(input->kafka->ctx, (uint8_t **) &buf, &buf_len); + res = kafka_read_start(input->kafka->ctx, &ubuf_str, &buf_len); if (res != nmsg_res_success) { kafka_read_finish(input->kafka->ctx); return res; } + buf = (char *)ubuf_str; + if (buf_len == 0) return nmsg_res_failure; - res = nmsg_message_from_json((const char *) buf, msg); + res = nmsg_message_from_json((const char *)buf, msg); if (res == nmsg_res_parse_error) { _nmsg_dprintf(2, "Kafka JSON parse error: \"%s\"\n", buf); From 384c36a83a7f7fd1f98759633f551fb8f224f501 Mon Sep 17 00:00:00 2001 From: regalk <72028266+regalk13@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:27:59 -0500 Subject: [PATCH 02/18] Make json-c build dependency mandatory --- README.md | 3 --- configure.ac | 10 +--------- nmsg/base/encode.c | 6 ------ nmsg/input.c | 13 +++---------- nmsg/input_json.c | 12 ++---------- nmsg/msgmod/message.c | 8 -------- nmsg/msgmod/transparent_json.c | 19 ------------------- nmsg/output.c | 2 -- nmsg/private.h | 4 ---- src/io.c | 30 +++++------------------------- src/nmsgtool.c | 18 +++--------------- src/process_args.c | 4 ++-- tests/test-private.c | 8 ++++---- 13 files changed, 20 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index 21527403..31bb4ff8 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,6 @@ to the `configure` script. Support for `librdkafka` can be disabled by passing the `--without-librdkafka` parameter to the `configure` script. -Support for `json-c` can be disabled by passing the `--without-json-c` parameter -to the `configure` script. - The documentation for the `libnmsg` API is located in the `doc/doxygen/html` directory. To rebuild the API documentation, run `make html`. This requires Doxygen to be installed. diff --git a/configure.ac b/configure.ac index 950ea322..1573e387 100644 --- a/configure.ac +++ b/configure.ac @@ -168,14 +168,7 @@ else use_libzmq="false" fi -AC_ARG_WITH([json-c], AS_HELP_STRING([--without-json-c], [Disable json-c support])) -if test "x$with_json_c" != "xno"; then - PKG_CHECK_MODULES([json_c], [json-c >= 0.13.0]) - AC_DEFINE([HAVE_JSON_C], [1], [Define to 1 if json-c support is enabled.]) - use_json_c="true" -else - use_json_c="false" -fi +PKG_CHECK_MODULES([json_c], [json-c >= 0.13.0]) AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ]) AC_CHECK_LIB([z], [deflate], [], [ AC_MSG_ERROR([required library not found]) ]) @@ -270,7 +263,6 @@ AC_MSG_RESULT([ bigendian: ${ac_cv_c_bigendian} libzmq support: ${use_libzmq} librdkafka support: ${use_librdkafka} - json-c support: ${use_json_c} building html docs: ${DOC_HTML_MSG} building manpage docs: ${DOC_MAN_MSG} diff --git a/nmsg/base/encode.c b/nmsg/base/encode.c index 8742800b..49974b59 100644 --- a/nmsg/base/encode.c +++ b/nmsg/base/encode.c @@ -18,9 +18,7 @@ */ /* Import. */ -#ifdef HAVE_JSON_C #include -#endif #include "encode.pb-c.h" @@ -66,7 +64,6 @@ encode_payload_add_value(struct nmsg_strbuf *sb, int type_value, const char *dat /* validate json */ if (is_json) { -#ifdef HAVE_JSON_C struct json_tokener *jtok = json_tokener_new(); struct json_object *jobj; @@ -79,9 +76,6 @@ encode_payload_add_value(struct nmsg_strbuf *sb, int type_value, const char *dat json_tokener_free(jtok); if (jobj == NULL) return false; -#else - return false; -#endif } declare_json_value(sb, "val", true); diff --git a/nmsg/input.c b/nmsg/input.c index 1bfc1066..9debc7a5 100644 --- a/nmsg/input.c +++ b/nmsg/input.c @@ -38,7 +38,7 @@ nmsg_input_open_sock(int fd) { return (input_open_stream(nmsg_stream_type_sock, fd)); } -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) nmsg_input_t nmsg_input_open_kafka_json(const char *address) { @@ -66,12 +66,12 @@ nmsg_input_open_kafka_json(const char *address) return (input); } -#else /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#else /* (defined HAVE_LIBRDKAFKA) */ nmsg_input_t nmsg_input_open_kafka_json(const char *address __attribute__((unused))) { return (NULL); } -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ #ifdef HAVE_LIBRDKAFKA nmsg_input_t @@ -178,7 +178,6 @@ nmsg_input_open_pres(int fd, nmsg_msgmod_t msgmod) { return (input); } -#ifdef HAVE_JSON_C nmsg_input_t nmsg_input_open_json(int fd) { struct nmsg_input *input; @@ -214,12 +213,6 @@ nmsg_input_open_json(int fd) { return (input); } -#else /* HAVE_JSON_C */ -nmsg_input_t -nmsg_input_open_json(__attribute__((unused)) int fd) { - return (NULL); -} -#endif /* HAVE_JSON_C */ nmsg_input_t nmsg_input_open_pcap(nmsg_pcap_t pcap, nmsg_msgmod_t msgmod) { diff --git a/nmsg/input_json.c b/nmsg/input_json.c index 0a5a8497..1e5d227e 100644 --- a/nmsg/input_json.c +++ b/nmsg/input_json.c @@ -21,7 +21,7 @@ /* Internal functions. */ -#if (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) +#if (defined HAVE_LIBRDKAFKA) nmsg_res _input_kafka_json_read(nmsg_input_t input, nmsg_message_t *msg) { uint8_t *ubuf_str; @@ -50,9 +50,8 @@ _input_kafka_json_read(nmsg_input_t input, nmsg_message_t *msg) { kafka_read_finish(input->kafka->ctx); return res; } -#endif /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ -#ifdef HAVE_JSON_C nmsg_res _input_json_read(nmsg_input_t input, nmsg_message_t *msg) { char line[1024]; @@ -97,10 +96,3 @@ _input_json_read(nmsg_input_t input, nmsg_message_t *msg) { _nmsg_strbuf_destroy(&sbs); return (nmsg_res_eof); } -#else /* HAVE_JSON_C */ -nmsg_res -_input_json_read(__attribute__((unused)) nmsg_input_t input, - __attribute__((unused)) nmsg_message_t *msg) { - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ diff --git a/nmsg/msgmod/message.c b/nmsg/msgmod/message.c index 8110d48f..34c94aa9 100644 --- a/nmsg/msgmod/message.c +++ b/nmsg/msgmod/message.c @@ -216,7 +216,6 @@ nmsg_message_from_raw_payload(unsigned vid, unsigned msgtype, return (msg); } -#ifdef HAVE_JSON_C nmsg_res nmsg_message_from_json(const char *json, nmsg_message_t *msg) { nmsg_res res = nmsg_res_parse_error; @@ -361,13 +360,6 @@ nmsg_message_from_json(const char *json, nmsg_message_t *msg) { json_object_put(node); return (res); } -#else /* HAVE_JSON_C */ -nmsg_res -nmsg_message_from_json(__attribute__((unused)) const char *json, - __attribute__((unused)) nmsg_message_t *msg) { - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ nmsg_res _nmsg_message_init_message(struct nmsg_message *msg) { diff --git a/nmsg/msgmod/transparent_json.c b/nmsg/msgmod/transparent_json.c index 08e57116..df3c3146 100644 --- a/nmsg/msgmod/transparent_json.c +++ b/nmsg/msgmod/transparent_json.c @@ -19,7 +19,6 @@ #include "transparent.h" -#ifdef HAVE_JSON_C nmsg_res _nmsg_msgmod_json_to_message(void *val, struct nmsg_message *msg) { struct json_object *node = (struct json_object *)val; @@ -321,21 +320,3 @@ _nmsg_msgmod_json_to_payload_load(struct nmsg_message *msg, return nmsg_res_failure; } - -#else /* HAVE_JSON_C */ -nmsg_res -_nmsg_msgmod_json_to_message(__attribute__((unused)) void *val, - __attribute__((unused)) struct nmsg_message *msg) { - return (nmsg_res_notimpl); -} - -nmsg_res -_nmsg_msgmod_json_to_payload_load(__attribute__((unused)) struct nmsg_message *msg, - __attribute__((unused)) struct nmsg_msgmod_field *field, - __attribute__((unused)) unsigned field_idx, - __attribute__((unused)) unsigned val_idx, - __attribute__((unused)) void *val) -{ - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ diff --git a/nmsg/output.c b/nmsg/output.c index 5321f819..744f5669 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -423,10 +423,8 @@ void _output_stop(nmsg_output_t output) { output->stop = true; #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C if (output->type == nmsg_output_type_kafka_json) kafka_stop(output->kafka->ctx); -#endif /* HAVE_JSON_C */ if (output->type == nmsg_output_type_stream && output->stream != NULL && output->stream->type == nmsg_stream_type_kafka) diff --git a/nmsg/private.h b/nmsg/private.h index bcff5b57..125752a8 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -66,9 +66,7 @@ #include #endif /* HAVE_LIBRDKAFKA */ -#ifdef HAVE_JSON_C #include -#endif /* HAVE_JSON_C */ #include "nmsg.h" #include "nmsg.pb-c.h" @@ -253,8 +251,6 @@ struct nmsg_pres { /* nmsg_json: used by nmsg_input and nmsg_output */ struct nmsg_json { -#ifdef HAVE_JSON_C -#endif /* HAVE_JSON_C */ pthread_mutex_t lock; FILE *fp; int orig_fd; diff --git a/src/io.c b/src/io.c index 2b11cc45..1cf96392 100644 --- a/src/io.c +++ b/src/io.c @@ -209,7 +209,7 @@ add_sock_output(nmsgtool_ctx *c, const char *ss) { } } -#if (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) +#if (defined HAVE_LIBRDKAFKA) static void _add_kafka_json_input(nmsgtool_ctx *c, const char *str_address) { nmsg_input_t input; @@ -233,15 +233,15 @@ _add_kafka_json_input(nmsgtool_ctx *c, const char *str_address) { str_address); c->n_inputs += 1; } -#else /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#else /* (defined HAVE_LIBRDKAFKA) */ static void _add_kafka_json_input(nmsgtool_ctx *c __attribute__((unused)), const char *str_address __attribute__((unused))) { - fprintf(stderr, "%s: Error: compiled without librdkafka or json-c support\n", + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", argv_program); exit(EXIT_FAILURE); } -#endif /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ #ifdef HAVE_LIBRDKAFKA static void @@ -273,7 +273,7 @@ _add_kafka_json_output(nmsgtool_ctx *c, const char *str_address) { static void _add_kafka_json_output(nmsgtool_ctx *c __attribute__((unused)), const char *str_address __attribute__((unused))) { - fprintf(stderr, "%s: Error: compiled without librdkafka or json-c support\n", + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", argv_program); exit(EXIT_FAILURE); } @@ -351,7 +351,6 @@ add_kafka_input(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_input(c, addr); return; } -#ifdef HAVE_JSON_C addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_input(c, addr); @@ -359,10 +358,6 @@ add_kafka_input(nmsgtool_ctx *c, const char *str_address) { } fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", argv_program); -#else /* HAVE_JSON_C */ - fprintf(stderr, "%s: Error: nmsg protocol must be set for Kafka topic\n", - argv_program); -#endif /* HAVE_JSON_C */ exit(EXIT_FAILURE); } @@ -373,7 +368,6 @@ add_kafka_output(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_output(c, addr); return; } -#ifdef HAVE_JSON_C addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_output(c, addr); @@ -381,10 +375,6 @@ add_kafka_output(nmsgtool_ctx *c, const char *str_address) { } fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", argv_program); -#else /* HAVE_JSON_C */ - fprintf(stderr, "%s: Error: nmsg protocol must be set for Kafka topic\n", - argv_program); -#endif /* HAVE_JSON_C */ exit(EXIT_FAILURE); } @@ -744,7 +734,6 @@ add_pres_output(nmsgtool_ctx *c, const char *fname) { c->n_outputs += 1; } -#ifdef HAVE_JSON_C void add_json_input(nmsgtool_ctx *c, const char *fname) { nmsg_input_t input; @@ -762,15 +751,6 @@ add_json_input(nmsgtool_ctx *c, const char *fname) { fname); c->n_inputs += 1; } -#else /* HAVE_JSON_C */ -void -add_json_input(__attribute__((unused)) nmsgtool_ctx *c, - __attribute__((unused)) const char *fname) { - fprintf(stderr, "%s: Error: compiled without json-c support\n", - argv_program); - exit(EXIT_FAILURE); -} -#endif /* HAVE_JSON_C */ void add_json_output(nmsgtool_ctx *c, const char *fname) { diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 638c14d8..f3d39f02 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -125,11 +125,7 @@ static argv_t args[] = { ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.r_json, "file", -#ifdef HAVE_JSON_C "read json format data from file" }, -#else /* HAVE_JSON_C */ - "read json format data from file (no support)" }, -#endif /* HAVE_JSON_C */ { 'J', "writejson", ARGV_CHAR_P | ARGV_FLAG_ARRAY, @@ -146,11 +142,11 @@ static argv_t args[] = { ARGV_CHAR_P, &ctx.kafka_key_field, "fieldname", -#if defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) +#if defined(HAVE_LIBRDKAFKA) "nmsg field for Kafka producer key" }, -#else /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#else /* defined(HAVE_LIBRDKAFKA) */ "nmsg field for Kafka producer key (no support)" }, -#endif /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#endif /* defined(HAVE_LIBRDKAFKA) */ {'\0', "readkafka", @@ -158,11 +154,7 @@ static argv_t args[] = { &ctx.r_kafka, "kafka", #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C "read nmsg data from Kafka (binary or json)" }, -#else /* HAVE_JSON_C */ - "read nmsg containers from Kafka topic" }, -#endif /* HAVE_JSON_C */ #else /* HAVE_LIBRDKAFKA */ "read nmsg data from Kafka topic (no support)" }, #endif /* HAVE_LIBRDKAFKA */ @@ -319,11 +311,7 @@ static argv_t args[] = { &ctx.w_kafka, "kafka", #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C "write nmsg data to Kafka (binary or json)" }, -#else /* HAVE_JSON_C */ - "write nmsg containers to to Kafka topic" }, -#endif /* HAVE_JSON_C */ #else /* HAVE_LIBRDKAFKA */ "write nmsg data to Kafka topic (no support)" }, #endif /* HAVE_LIBRDKAFKA */ diff --git a/src/process_args.c b/src/process_args.c index 472adfe6..38f9f6a5 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -175,14 +175,14 @@ process_args(nmsgtool_ctx *c) { c->kicker = strdup(t); } -#if defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) +#if defined(HAVE_LIBRDKAFKA) /* kafka key */ if (c->kafka_key_field == NULL) { t = getenv("NMSG_KAFKA_KEY"); if (t != NULL) c->kafka_key_field = t; } -#endif /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#endif /* defined(HAVE_LIBRDKAFKA) */ /* set source, operator, group */ if (c->set_source_str != NULL) { diff --git a/tests/test-private.c b/tests/test-private.c index 55a3bd38..788e3548 100644 --- a/tests/test-private.c +++ b/tests/test-private.c @@ -32,7 +32,7 @@ typedef int (*config_test)(struct config_file *); -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) typedef struct { const char *field; size_t length; @@ -178,7 +178,7 @@ test_kafka_key(void) { l_return_test_status(); } -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ static int _test_config_file_papi_null(void) { @@ -337,10 +337,10 @@ main(void) check_explicit2_display_only(test_config_file() == 0, "test-private / test_config_file"); -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) check_explicit2_display_only(test_kafka_papi() == 0, "test-private / test_kafka_papi"); check_explicit2_display_only(test_kafka_key() == 0, "test-private / test_kafka_key"); -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ g_check_test_status(false); } From dae804735213e2f954d688a1f3cc3f5e2957b09e Mon Sep 17 00:00:00 2001 From: "Jeremy C. Reed" Date: Fri, 17 Apr 2026 16:51:20 +0000 Subject: [PATCH 03/18] print calculated crc in host byte order for mismatch warning debug message it is compared using host byte order but previously was printed as is (network byte order) --- nmsg/input_nmsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmsg/input_nmsg.c b/nmsg/input_nmsg.c index e67e9c91..5d2d2774 100644 --- a/nmsg/input_nmsg.c +++ b/nmsg/input_nmsg.c @@ -156,7 +156,7 @@ _input_nmsg_filter(nmsg_input_t input, unsigned idx, Nmsg__NmsgPayload *np) { uint32_t calc_crc = my_crc32c(np->payload.data, np->payload.len); if (ntohl(wire_crc) != calc_crc) { _nmsg_dprintf(1, "libnmsg: WARNING: crc mismatch (%x != %x) [%s]\n", - calc_crc, wire_crc, __func__); + calc_crc, ntohl(wire_crc), __func__); return (false); } } From 00c3bfedce4deac59dcb7d2f91179a5837a33c1e Mon Sep 17 00:00:00 2001 From: Rafael Vanoni Date: Wed, 27 May 2026 16:54:15 -0700 Subject: [PATCH 04/18] nmsg --readpres hang for base email and logline and linkpair --- doc/docbook/nmsgtool.1 | 11 ----------- doc/docbook/nmsgtool.docbook | 15 --------------- src/io.c | 18 ------------------ src/nmsgtool.c | 6 ------ src/nmsgtool.h | 3 +-- src/process_args.c | 9 +++------ tests/nmsg-dns-tests/test.sh.in | 2 -- tests/nmsg-dnsobs-tests/test.sh.in | 2 -- tests/nmsg-dnsqr-tests/test.sh.in | 2 -- tests/nmsg-dnstap-tests/test.sh.in | 1 - tests/nmsg-http-tests/test.sh.in | 2 -- 11 files changed, 4 insertions(+), 67 deletions(-) diff --git a/doc/docbook/nmsgtool.1 b/doc/docbook/nmsgtool.1 index 0b7bad9d..7a1633e5 100644 --- a/doc/docbook/nmsgtool.1 +++ b/doc/docbook/nmsgtool.1 @@ -355,17 +355,6 @@ environment variable\&. Read NMSG payloads from a file\&. .RE .PP -\fB\-f\fR \fIfile\fR, \fB\-\-readpres\fR \fIfile\fR -.RS 4 -Read presentation format data from a file and convert to NMSG payloads\&. This option is dependent on the -\fB\-V\fR -and -\fB\-T\fR -options being set in order to select a specific nmsgpb module to perform presentation format to NMSG payload conversion\&. Not all nmsgpb modules necessarily support this conversion method, in which case -\fBnmsgtool\fR -will print a "function not implemented" message\&. -.RE -.PP \fB\-j\fR \fIfile\fR, \fB\-\-readjson\fR \fIfile\fR .RS 4 Read JSON format data from a file\&. See documentation for diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 6e0e5901..96304bf2 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -401,21 +401,6 @@ - - file - file - - Read presentation format data from a file and convert - to NMSG payloads. This option is dependent on the - and options being - set in order to select a specific nmsgpb module to perform - presentation format to NMSG payload conversion. Not all nmsgpb - modules necessarily support this conversion method, in which - case nmsgtool will print a "function not - implemented" message. - - - file file diff --git a/src/io.c b/src/io.c index 1cf96392..d5cd8065 100644 --- a/src/io.c +++ b/src/io.c @@ -682,24 +682,6 @@ add_pcapif_input(nmsgtool_ctx *c, nmsg_msgmod_t mod, const char *arg) { free(tmp); } -void -add_pres_input(nmsgtool_ctx *c, nmsg_msgmod_t mod, const char *fname) { - nmsg_input_t input; - nmsg_res res; - - input = nmsg_input_open_pres(open_rfile(fname), mod); - res = nmsg_io_add_input(c->io, input, NULL); - if (res != nmsg_res_success) { - fprintf(stderr, "%s: nmsg_io_add_input() failed\n", - argv_program); - exit(1); - } - if (c->debug >= 2) - fprintf(stderr, "%s: nmsg pres input: %s\n", argv_program, - fname); - c->n_inputs += 1; -} - void add_pres_output(nmsgtool_ctx *c, const char *fname) { nmsg_output_t output; diff --git a/src/nmsgtool.c b/src/nmsgtool.c index f3d39f02..62a9c506 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -78,12 +78,6 @@ static argv_t args[] = { "endline", "continuation separator" }, - { 'f', "readpres", - ARGV_CHAR_P | ARGV_FLAG_ARRAY, - &ctx.r_pres, - "file", - "read pres format data from file" }, - { 'F', "filter", ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.filters, diff --git a/src/nmsgtool.h b/src/nmsgtool.h index ef2ec41c..2d1c99b8 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -53,7 +53,7 @@ VECTOR_GENERATE(statsmod_vec, nmsg_statsmod_t) typedef struct { /* parameters */ argv_array_t filters, statsmods; - argv_array_t r_nmsg, r_pres, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; + argv_array_t r_nmsg, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; argv_array_t r_pcapfile, r_pcapif; argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; @@ -125,7 +125,6 @@ void add_file_input(nmsgtool_ctx *, const char *); void add_file_output(nmsgtool_ctx *, const char *); void add_pcapfile_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pcapif_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); -void add_pres_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pres_output(nmsgtool_ctx *, const char *); void add_json_input(nmsgtool_ctx *, const char *); void add_json_output(nmsgtool_ctx *, const char *); diff --git a/src/process_args.c b/src/process_args.c index 38f9f6a5..c9c49419 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -263,13 +263,11 @@ process_args(nmsgtool_ctx *c) { } /* -V, -T sanity check */ - if (ARGV_ARRAY_COUNT(c->r_pres) > 0 || - ARGV_ARRAY_COUNT(c->r_pcapfile) > 0 || + if (ARGV_ARRAY_COUNT(c->r_pcapfile) > 0 || ARGV_ARRAY_COUNT(c->r_pcapif) > 0) { if (c->vname == NULL || c->mname == NULL) - usage("reading presentation or pcap data requires " - "-V, -T"); + usage("reading pcap data requires -V, -T"); mod = nmsg_msgmod_lookup(c->vid, c->msgtype); if (mod == NULL) usage("unknown msgmod"); @@ -368,8 +366,7 @@ process_args(nmsgtool_ctx *c) { nmsg_chalias_free(&alias); } - /* pres inputs and outputs */ - process_args_loop_mod(c->r_pres, add_pres_input, mod); + /* pres outputs */ process_args_loop(c->w_pres, add_pres_output); /* json inputs and outputs */ diff --git a/tests/nmsg-dns-tests/test.sh.in b/tests/nmsg-dns-tests/test.sh.in index 24f76f77..766aaa1b 100755 --- a/tests/nmsg-dns-tests/test.sh.in +++ b/tests/nmsg-dns-tests/test.sh.in @@ -76,8 +76,6 @@ check read json base:dns and create json output cmp -s @abs_top_srcdir@/tests/nmsg-dns-tests/test2-dns.json @abs_top_builddir@/tests/nmsg-dns-tests/test2-dns.json.json.out check json-to-json -# NOTE: --readpres is not fully implemented for base:dns so aborts - # JSON input mistakes should result in no output $NMSGTOOL -dd -j @abs_top_srcdir@/tests/nmsg-dns-tests/test3-dns.json --writepres @abs_top_builddir@/tests/nmsg-dns-tests/test3-dns.json.pres.out 2>@abs_top_builddir@/tests/nmsg-dns-tests/test3-dns.json.pres.stderr.out check read broken json base:dns and create empty output diff --git a/tests/nmsg-dnsobs-tests/test.sh.in b/tests/nmsg-dnsobs-tests/test.sh.in index 0ae93a11..04a0186b 100755 --- a/tests/nmsg-dnsobs-tests/test.sh.in +++ b/tests/nmsg-dnsobs-tests/test.sh.in @@ -54,6 +54,4 @@ check read json base:dnsobs and create base:dnsobs json output cmp -s ${SOURCE}.json ${OUTPUT}.json.json.out check json-to-json -# NOTE: --readpres is not fully implemented for base:dnsobs - exit $status diff --git a/tests/nmsg-dnsqr-tests/test.sh.in b/tests/nmsg-dnsqr-tests/test.sh.in index 4cbeb3c1..20f83311 100755 --- a/tests/nmsg-dnsqr-tests/test.sh.in +++ b/tests/nmsg-dnsqr-tests/test.sh.in @@ -84,6 +84,4 @@ check read nmsg base:dnsqr and generate pcap output using example cmp -s ${SOURCE}.pcap ${OUTPUT}.nmsg.pcap.out check example-nmsg-to-pcap -# NOTE: --readpres is not fully implemented for base:dnsqr so aborts - exit $status diff --git a/tests/nmsg-dnstap-tests/test.sh.in b/tests/nmsg-dnstap-tests/test.sh.in index 08093574..eea0adeb 100755 --- a/tests/nmsg-dnstap-tests/test.sh.in +++ b/tests/nmsg-dnstap-tests/test.sh.in @@ -57,6 +57,5 @@ cmp -s ${SOURCE}.json ${OUTPUT}.nmsg.json.out check nmsg-to-json # NOTE: --readjson for base:dnstap is incomplete -# NOTE: --readpres is not fully implemented for base:dnstap exit $status diff --git a/tests/nmsg-http-tests/test.sh.in b/tests/nmsg-http-tests/test.sh.in index 812dba7e..0084e719 100755 --- a/tests/nmsg-http-tests/test.sh.in +++ b/tests/nmsg-http-tests/test.sh.in @@ -136,6 +136,4 @@ check read json base:http and create base:http json output cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.json.json.out check json-to-json -# NOTE: --readpres is not implemented for base:http - exit $status From e0833f6fc41a54b8217b0f400fd3772082a7cfae Mon Sep 17 00:00:00 2001 From: Rafael Vanoni Date: Fri, 29 May 2026 03:38:55 -0700 Subject: [PATCH 05/18] fix loop in _nmsg_strbuf_expand() --- nmsg/strbuf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nmsg/strbuf.c b/nmsg/strbuf.c index 2749bba5..ece4f644 100644 --- a/nmsg/strbuf.c +++ b/nmsg/strbuf.c @@ -69,10 +69,9 @@ _nmsg_strbuf_expand(struct nmsg_strbuf *sb, size_t len) { ssize_t avail = _nmsg_strbuf_avail(sb); assert(avail >= 0); - /* increase buffer size if necessary */ if (needed > avail) { size_t offset = sb->pos - sb->data; - ssize_t new_bufsz = 2 * sb->bufsz; + ssize_t new_bufsz = 2 * (sb->bufsz == 0 ? 1 : sb->bufsz); void *ptr; while (new_bufsz - (ssize_t) sb->bufsz < needed) { From 307e00b7955251025e67253b5e9280eb83e9b65d Mon Sep 17 00:00:00 2001 From: Rafael Vanoni Date: Thu, 7 May 2026 21:26:52 -0700 Subject: [PATCH 06/18] check return of alloc/dup() calls; check max buf size and fix offset in zlib inflate() --- nmsg/alias.c | 4 ++++ nmsg/chalias.c | 36 +++++++++++++++++++++++++++++------- nmsg/chalias.h | 3 ++- nmsg/dlmod.c | 1 + nmsg/output.c | 21 ++++++++++++++++++--- nmsg/pcap_input.c | 8 ++++++++ nmsg/zbuf.c | 6 +++++- 7 files changed, 67 insertions(+), 12 deletions(-) diff --git a/nmsg/alias.c b/nmsg/alias.c index eae9ee91..03168853 100644 --- a/nmsg/alias.c +++ b/nmsg/alias.c @@ -169,6 +169,10 @@ alias_init(struct nmsg_alias *al, const char *fname) { } al->value[key] = strdup(str_value); + if (al->value[key] == NULL) { + res = nmsg_res_failure; + break; + } } fclose(fp); diff --git a/nmsg/chalias.c b/nmsg/chalias.c index 87082083..d1c72b38 100644 --- a/nmsg/chalias.c +++ b/nmsg/chalias.c @@ -61,13 +61,28 @@ nmsg_chalias_lookup(const char *ch, char ***alias) { while (fgets(line, sizeof(line), fp) != NULL) { tmp = strtok_r(line, " \t", &saveptr); if (tmp != NULL && strcmp(tmp, ch) == 0) { - while ((tmp = strtok_r(NULL, " \t\n", &saveptr)) - != NULL) - { + while ((tmp = strtok_r(NULL, " \t\n", &saveptr)) != NULL) { + char **ptr, *dup; + num_aliases += 1; - *alias = realloc(*alias, - sizeof(**alias) * num_aliases); - (*alias)[num_aliases - 1] = strdup(tmp); + + ptr = realloc(*alias, sizeof(**alias) * num_aliases); + if (ptr == NULL) { + fclose(fp); + return (-1); + } + + *alias = ptr; + + dup = strdup(tmp); + + // terminate the alias buffer whether strdup(3) succeeds or not + (*alias)[num_aliases - 1] = dup; + + if (dup == NULL) { + fclose(fp); + return (-1); + } } } } @@ -75,7 +90,14 @@ nmsg_chalias_lookup(const char *ch, char ***alias) { fclose(fp); /* append NULL sentinel */ - *alias = realloc(*alias, sizeof(**alias) * (num_aliases + 1)); + char **ptr = realloc(*alias, sizeof(**alias) * (num_aliases + 1)); + if (ptr == NULL && *alias != NULL) { + free((*alias)[num_aliases - 1]); + (*alias)[num_aliases - 1] = NULL; + return (-1); + } + + *alias = ptr; (*alias)[num_aliases] = NULL; return (num_aliases); diff --git a/nmsg/chalias.h b/nmsg/chalias.h index 73b9164b..eef7e2c2 100644 --- a/nmsg/chalias.h +++ b/nmsg/chalias.h @@ -28,7 +28,8 @@ * * \param[out] alias Location to store an array of sockspecs. * - * \return Number of aliases. + * \return Number of aliases or -1 if an error has occurred, in which case + * the caller must invoke nmsg_chalias_free() for the alias. */ int nmsg_chalias_lookup(const char *ch, char ***alias); diff --git a/nmsg/dlmod.c b/nmsg/dlmod.c index c316523d..6d23a667 100644 --- a/nmsg/dlmod.c +++ b/nmsg/dlmod.c @@ -35,6 +35,7 @@ _nmsg_dlmod_init(const char *path) { dlmod->handle = dlopen(path, RTLD_LAZY); if (dlmod->handle == NULL) { _nmsg_dprintf(1, "%s: %s\n", __func__, dlerror()); + free(dlmod->path); free(dlmod); return (NULL); } diff --git a/nmsg/output.c b/nmsg/output.c index 744f5669..cf3e0d91 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -64,8 +64,15 @@ nmsg_output_open_kafka_json(const char *addr, const char *key_field) return NULL; } - if (key_field != NULL) + if (key_field != NULL) { output->kafka->key_field = strdup(key_field); + if (output->kafka->key_field == NULL) { + kafka_ctx_destroy(&output->kafka->ctx); + free(output->kafka); + free(output); + return (NULL); + } + } return output; }; @@ -137,6 +144,12 @@ nmsg_output_open_pres(int fd) { return (NULL); } output->pres->endline = strdup("\n"); + if (output->pres->endline == NULL) { + fclose(output->pres->fp); + free(output->pres); + free(output); + return (NULL); + } pthread_mutex_init(&output->pres->lock, NULL); return (output); @@ -356,9 +369,11 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { void nmsg_output_set_endline(nmsg_output_t output, const char *endline) { if (output->type == nmsg_output_type_pres) { - if (output->pres->endline != NULL) + char *ptr = strdup(endline); + if (ptr != NULL) { free(output->pres->endline); - output->pres->endline = strdup(endline); + output->pres->endline = ptr; + } } } diff --git a/nmsg/pcap_input.c b/nmsg/pcap_input.c index d7bba7e0..61e114cc 100644 --- a/nmsg/pcap_input.c +++ b/nmsg/pcap_input.c @@ -160,6 +160,10 @@ nmsg_pcap_input_setfilter_raw(nmsg_pcap_t pcap, const char *userbpft) { return (nmsg_res_failure); } pcap->userbpft = strdup(userbpft); + if (pcap->userbpft == NULL) { + pcap_freecode(&pcap->userbpf); + return (nmsg_res_memfail); + } /* test if we can skip vlan tags */ res = pcap_compile(pcap->handle, &bpf, "vlan and ip", 1, 0); @@ -241,6 +245,10 @@ nmsg_pcap_input_setfilter(nmsg_pcap_t pcap, const char *userbpft) { return (nmsg_res_failure); } pcap->userbpft = strdup(userbpft); + if (pcap->userbpft == NULL) { + pcap_freecode(&pcap->userbpf); + return (nmsg_res_memfail); + } /* test if we can skip ip6 */ res = nmsg_asprintf(&tmp, "(%s) and %s", userbpft, bpf_ip6); diff --git a/nmsg/zbuf.c b/nmsg/zbuf.c index 4af73dbd..713eea3f 100644 --- a/nmsg/zbuf.c +++ b/nmsg/zbuf.c @@ -126,6 +126,9 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, uint32_t my_ulen; load_net32(z_buf, &my_ulen); + if (my_ulen > NMSG_WBUFSZ_MAX) + return (nmsg_res_memfail); + z_buf += 4; *u_len = my_ulen; @@ -133,7 +136,7 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, if (*u_buf == NULL) return (nmsg_res_memfail); - zb->zs.avail_in = z_len; + zb->zs.avail_in = z_len - 4; zb->zs.next_in = z_buf; zb->zs.avail_out = *u_len; zb->zs.next_out = *u_buf; @@ -142,6 +145,7 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, if (zret != Z_STREAM_END || zb->zs.avail_out != 0) { _nmsg_dprintf(1, "%s: inflate() failed\n", __func__); free(*u_buf); + *u_buf = NULL; return (nmsg_res_failure); } zret = inflateReset(&zb->zs); From 4cc5f38720d112ebc4443e3b82678e8c561c55a6 Mon Sep 17 00:00:00 2001 From: Rafael Vanoni Date: Fri, 15 May 2026 17:32:31 -0700 Subject: [PATCH 07/18] remove unused libmy functions/files --- libmy/b32_decode.c | 189 ----------------------------------------- libmy/b32_decode.h | 52 ------------ libmy/b32_encode.c | 127 ---------------------------- libmy/b32_encode.h | 52 ------------ libmy/heap.c | 158 ---------------------------------- libmy/heap.h | 17 ---- libmy/spooldir.c | 205 --------------------------------------------- libmy/spooldir.h | 10 --- libmy/varint.c | 189 ----------------------------------------- libmy/varint.h | 14 ---- libmy/zonefile.c | 188 ----------------------------------------- libmy/zonefile.h | 26 ------ 12 files changed, 1227 deletions(-) delete mode 100644 libmy/b32_decode.c delete mode 100644 libmy/b32_decode.h delete mode 100644 libmy/b32_encode.c delete mode 100644 libmy/b32_encode.h delete mode 100644 libmy/heap.c delete mode 100644 libmy/heap.h delete mode 100644 libmy/spooldir.c delete mode 100644 libmy/spooldir.h delete mode 100644 libmy/varint.c delete mode 100644 libmy/varint.h delete mode 100644 libmy/zonefile.c delete mode 100644 libmy/zonefile.h diff --git a/libmy/b32_decode.c b/libmy/b32_decode.c deleted file mode 100644 index 53038114..00000000 --- a/libmy/b32_decode.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2015 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * See RFC 4648 for details about Base 32 hex encoding: - * http://tools.ietf.org/html/rfc4648 - */ - -#include -#include - -#include "b32_decode.h" - -#ifndef G_N_ELEMENTS -#define G_N_ELEMENTS(arr) (sizeof (arr) / sizeof ((arr)[0])) -#endif - -#define ZERO(x) memset((x), 0, sizeof *(x)) - -static inline void * -ptr_add_offset(void *p, size_t offset) -{ - /* Using size_t instead of 'char *' because pointers don't wrap. */ - return (void *) ((size_t) p + offset); -} - -static inline const void * -cast_to_constpointer(const void *p) -{ - return p; -} - -static inline bool -is_ascii_lower(int c) -{ - return c >= 97 && c <= 122; /* a-z */ -} - -static inline int -ascii_toupper(int c) -{ - return is_ascii_lower(c) ? c - 32 : c; -} - -static inline size_t -ptr_diff(const void *a, const void *b) -{ - return (const char *) a - (const char *) b; -} - -static const char base32_alphabet[32] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V' -}; - -static char base32_map[(size_t) (unsigned char) -1 + 1]; - -/** - * Decode a base32 encoding of `len' bytes of `data' into the buffer `dst'. - * - * @param dst destination buffer - * @param size length of destination - * @param data start of data to decode - * @param len amount of encoded data to decode - * - * @return the amount of bytes decoded into the destination. - */ -size_t -base32_decode(void *dst, size_t size, const char *data, size_t len) -{ - const char *end = ptr_add_offset(dst, size); - const unsigned char *p = cast_to_constpointer(data); - char s[8]; - char *q = dst; - int pad = 0; - size_t i, si; - - if (0 == base32_map[0]) { - for (i = 0; i < G_N_ELEMENTS(base32_map); i++) { - const char *x; - - x = memchr(base32_alphabet, ascii_toupper(i), - sizeof base32_alphabet); - base32_map[i] = x ? (x - base32_alphabet) : (unsigned char) -1; - } - } - - ZERO(&s); - si = 0; - i = 0; - - while (i < len) { - unsigned char c; - - c = p[i++]; - if ('=' == c) { - pad++; - c = 0; - } else { - c = base32_map[c]; - if ((unsigned char) -1 == c) { - return -1; - } - } - - s[si++] = c; - - if (G_N_ELEMENTS(s) == si || pad > 0 || i == len) { - char b[5]; - size_t bi; - - memset(&s[si], 0, G_N_ELEMENTS(s) - si); - si = 0; - - b[0] = - ((s[0] << 3) & 0xf8) | - ((s[1] >> 2) & 0x07); - b[1] = - ((s[1] & 0x03) << 6) | - ((s[2] & 0x1f) << 1) | - ((s[3] >> 4) & 1); - b[2] = - ((s[3] & 0x0f) << 4) | - ((s[4] >> 1) & 0x0f); - b[3] = - ((s[4] & 1) << 7) | - ((s[5] & 0x1f) << 2) | - ((s[6] >> 3) & 0x03); - b[4] = - ((s[6] & 0x07) << 5) | - (s[7] & 0x1f); - - for (bi = 0; bi < G_N_ELEMENTS(b) && q != end; bi++) { - *q++ = b[bi]; - } - } - - if (end == q) { - break; - } - } - - return ptr_diff(q, dst); -} diff --git a/libmy/b32_decode.h b/libmy/b32_decode.h deleted file mode 100644 index bc8d4ad9..00000000 --- a/libmy/b32_decode.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2015 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef B32_DECODE_H -#define B32_DECODE_H - -size_t base32_decode(void *dst, size_t size, const char *data, size_t len); - -#endif /* B32_DECODE_H */ diff --git a/libmy/b32_encode.c b/libmy/b32_encode.c deleted file mode 100644 index 6487fc44..00000000 --- a/libmy/b32_encode.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * See RFC 4648 for details about Base 32 hex encoding: - * http://tools.ietf.org/html/rfc4648 - */ - -#include -#include - -#include "b32_encode.h" - -static const char base32_alphabet[32] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V' -}; - -/** - * Encode in base32 `len' bytes of `data' into the buffer `dst'. - * - * @param dst destination buffer - * @param size length of destination - * @param data start of data to encode - * @param len amount of bytes to encode - * - * @return the amount of bytes generated into the destination. - */ -size_t -base32_encode(char *dst, size_t size, const void *data, size_t len) -{ - size_t i = 0; - const uint8_t *p = data; - const char *end = &dst[size]; - char *q = dst; - - do { - size_t j, k; - uint8_t x[5]; - char s[8]; - - switch (len - i) { - case 4: - k = 7; - break; - case 3: - k = 5; - break; - case 2: - k = 3; - break; - case 1: - k = 2; - break; - default: - k = 8; - } - - for (j = 0; j < 5; j++) - x[j] = i < len ? p[i++] : 0; - - s[0] = (x[0] >> 3); - s[1] = ((x[0] & 0x07) << 2) | (x[1] >> 6); - s[2] = (x[1] >> 1) & 0x1f; - s[3] = ((x[1] & 0x01) << 4) | (x[2] >> 4); - s[4] = ((x[2] & 0x0f) << 1) | (x[3] >> 7); - s[5] = (x[3] >> 2) & 0x1f; - s[6] = ((x[3] & 0x03) << 3) | (x[4] >> 5); - s[7] = x[4] & 0x1f; - - for (j = 0; j < k && q != end; j++) { - *q++ = base32_alphabet[(uint8_t) s[j]]; - } - - if (end == q) { - break; - } - - } while (i < len); - - return q - dst; -} diff --git a/libmy/b32_encode.h b/libmy/b32_encode.h deleted file mode 100644 index 2500f390..00000000 --- a/libmy/b32_encode.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef B32_ENCODE_H -#define B32_ENCODE_H - -size_t base32_encode(char *dst, size_t dst_len, const void *src, size_t src_len); - -#endif /* B32_ENCODE_H */ diff --git a/libmy/heap.c b/libmy/heap.c deleted file mode 100644 index 1d8f7ee6..00000000 --- a/libmy/heap.c +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2012 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "my_alloc.h" -#include "heap.h" -#include "vector.h" - -VECTOR_GENERATE(ptrvec, void *); - -struct heap { - ptrvec *vec; - heap_compare_func cmp; -}; - -static inline int -cmp_wrapper(heap_compare_func cmp, const void *a, const void *b) -{ - return ((cmp(a, b) < 0) ? 1 : 0); -} - -struct heap * -heap_init(heap_compare_func cmp) -{ - struct heap *h = my_calloc(1, sizeof(*h)); - h->cmp = cmp; - h->vec = ptrvec_init(1); - return (h); -} - -void -heap_destroy(struct heap **h) -{ - if (*h != NULL) { - ptrvec_destroy(&(*h)->vec); - free(*h); - *h = NULL; - } -} - -static int -siftdown(struct heap *h, size_t startpos, size_t pos) -{ - assert(pos < ptrvec_size(h->vec)); - void *newitem = ptrvec_value(h->vec, pos); - while (pos > startpos) { - size_t parentpos = (pos - 1) >> 1; - void *parent = ptrvec_value(h->vec, parentpos); - int cmp = cmp_wrapper(h->cmp, newitem, parent); - if (cmp == -1) - return (-1); - if (cmp == 0) - break; - ptrvec_data(h->vec)[pos] = parent; - pos = parentpos; - } - ptrvec_data(h->vec)[pos] = newitem; - return (0); -} - -static int -siftup(struct heap *h, size_t pos) -{ - assert(pos < ptrvec_size(h->vec)); - void *newitem = ptrvec_value(h->vec, pos); - size_t endpos = ptrvec_size(h->vec); - size_t startpos = pos; - size_t childpos = 2 * pos + 1; - while (childpos < endpos) { - size_t rightpos = childpos + 1; - if (rightpos < endpos) { - int cmp = cmp_wrapper(h->cmp, - ptrvec_value(h->vec, childpos), - ptrvec_value(h->vec, rightpos)); - if (cmp == -1) - return (-1); - if (cmp == 0) - childpos = rightpos; - } - ptrvec_data(h->vec)[pos] = ptrvec_value(h->vec, childpos); - pos = childpos; - childpos = 2 * pos + 1; - } - ptrvec_data(h->vec)[pos] = newitem; - return (siftdown(h, startpos, pos)); -} - -void -heap_push(struct heap *h, void *item) -{ - ptrvec_add(h->vec, item); - siftdown(h, 0, ptrvec_size(h->vec) - 1); -} - -void * -heap_pop(struct heap *h) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - void *returnitem; - void *lastelt = ptrvec_value(h->vec, ptrvec_size(h->vec) - 1); - ptrvec_clip(h->vec, ptrvec_size(h->vec) - 1); - if (ptrvec_size(h->vec) > 0) { - returnitem = ptrvec_value(h->vec, 0); - ptrvec_data(h->vec)[0] = lastelt; - siftup(h, 0); - } else { - returnitem = lastelt; - } - return (returnitem); -} - -void * -heap_replace(struct heap *h, void *item) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - void *returnitem = ptrvec_value(h->vec, 0); - ptrvec_data(h->vec)[0] = item; - siftup(h, 0); - return (returnitem); -} - -void * -heap_peek(struct heap *h) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - return ptrvec_data(h->vec)[0]; -} - -void * -heap_get(struct heap *h, size_t i) -{ - if (i > ptrvec_size(h->vec) - 1) - return (NULL); - return (ptrvec_value(h->vec, i)); -} - -size_t -heap_size(struct heap *h) -{ - return (ptrvec_size(h->vec)); -} diff --git a/libmy/heap.h b/libmy/heap.h deleted file mode 100644 index dcd0b0c1..00000000 --- a/libmy/heap.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef MY_HEAP_H -#define MY_HEAP_H - -struct heap; - -typedef int (*heap_compare_func)(const void *a, const void *b); - -struct heap *heap_init(heap_compare_func); -void heap_destroy(struct heap **); -void heap_push(struct heap *, void *); -void *heap_pop(struct heap *); -void *heap_replace(struct heap *, void *); -void *heap_peek(struct heap *); -void *heap_get(struct heap *, size_t); -size_t heap_size(struct heap *); - -#endif /* MY_HEAP_H */ diff --git a/libmy/spooldir.c b/libmy/spooldir.c deleted file mode 100644 index 6c7306de..00000000 --- a/libmy/spooldir.c +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2012 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "my_alloc.h" -#include "spooldir.h" -#include "ubuf.h" - -#define UBUFSZ 128 - -struct spooldir { - pthread_mutex_t lock; - DIR *dir; - int dir_fd; - ubuf *fname; - ubuf *dname_active; - ubuf *dname_incoming; -}; - -static bool -path_exists(const char *path) -{ - struct stat sb; - int ret; - - ret = stat(path, &sb); - if (ret < 0) - return (false); - return (true); -} - -static bool -path_isdir(const char *path) -{ - struct stat sb; - int ret; - - ret = stat(path, &sb); - if (ret < 0) - return (false); - if (S_ISDIR(sb.st_mode)) - return (true); - return (false); -} - -static bool -path_mkdir(const char *path, mode_t mode) -{ - int ret; - - if (path_isdir(path)) { - return (true); - } else { - ret = mkdir(path, mode); - if (ret < 0) { - perror("mkdir"); - return (false); - } - } - return (true); -} - -struct spooldir * -spooldir_init(const char *path) -{ - struct spooldir *s = my_calloc(1, sizeof(*s)); - bool res; - char *dname; - - pthread_mutex_init(&s->lock, NULL); - - dname = realpath(path, NULL); - assert(dname != NULL); - - assert(path_isdir(dname)); - - s->dname_active = ubuf_init(UBUFSZ); - ubuf_add_cstr(s->dname_active, dname); - ubuf_add_cstr(s->dname_active, "/active"); - res = path_mkdir(ubuf_cstr(s->dname_active), 0755); - assert(res); - - s->dname_incoming = ubuf_init(UBUFSZ); - ubuf_add_cstr(s->dname_incoming, dname); - ubuf_add_cstr(s->dname_incoming, "/incoming"); - res = path_mkdir(ubuf_cstr(s->dname_incoming), 0755); - assert(res); - - free(dname); - - s->dir = opendir(ubuf_cstr(s->dname_incoming)); - assert(s->dir != NULL); - - s->dir_fd = dirfd(s->dir); - assert(s->dir_fd != -1); - - s->fname = ubuf_init(UBUFSZ); - - return (s); -} - -void -spooldir_destroy(struct spooldir **s) -{ - if (*s != NULL) { - pthread_mutex_destroy(&(*s)->lock); - closedir((*s)->dir); - ubuf_destroy(&(*s)->fname); - ubuf_destroy(&(*s)->dname_active); - ubuf_destroy(&(*s)->dname_incoming); - free(*s); - *s = NULL; - } -} - -char * -spooldir_next(struct spooldir *s) -{ - struct stat sb; - struct dirent *de; - char *ret = NULL; - size_t retsz; - char *fname = NULL; - ubuf *src_fname; - - pthread_mutex_lock(&s->lock); - - while (fname == NULL) { - while ((de = readdir(s->dir)) != NULL) { - if (de->d_name[0] == '.') - continue; - if (fstatat(s->dir_fd, de->d_name, &sb, 0) == -1) { - fprintf(stderr, "%s: fstatat() failed: %s\n", - __func__, strerror(errno)); - continue; - } - if (!S_ISREG(sb.st_mode)) - continue; - fname = de->d_name; - break; - } - - if (fname == NULL) { - rewinddir(s->dir); - usleep(500*1000); - pthread_mutex_unlock(&s->lock); - usleep(500*1000); - return (NULL); - } - } - - assert(fname != NULL); - - src_fname = ubuf_init(UBUFSZ); - ubuf_extend(src_fname, s->dname_incoming); - ubuf_add_fmt(src_fname, "/%s", fname); - ubuf_cterm(src_fname); - - ubuf_clip(s->fname, 0); - ubuf_extend(s->fname, s->dname_active); - ubuf_add_fmt(s->fname, "/%s", fname); - ubuf_cterm(s->fname); - - if (path_exists(ubuf_cstr(s->fname))) { - fprintf(stderr, "%s: WARNING: unlinking destination path %s\n", - __func__, ubuf_cstr(s->fname)); - unlink(ubuf_cstr(s->fname)); - } - - int rename_ret = rename(ubuf_cstr(src_fname), ubuf_cstr(s->fname)); - if (rename_ret != 0) { - fprintf(stderr, "rename(%s, %s): %s\n", - ubuf_cstr(src_fname), ubuf_cstr(s->fname), strerror(errno)); - goto out; - } - - ubuf_detach(s->fname, (uint8_t **) &ret, &retsz); -out: - ubuf_destroy(&src_fname); - pthread_mutex_unlock(&s->lock); - return (ret); -} diff --git a/libmy/spooldir.h b/libmy/spooldir.h deleted file mode 100644 index 88d146df..00000000 --- a/libmy/spooldir.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef MY_SPOOLDIR_H -#define MY_SPOOLDIR_H - -struct spooldir; - -struct spooldir *spooldir_init(const char *path); -void spooldir_destroy(struct spooldir **); -char *spooldir_next(struct spooldir *); - -#endif /* MY_SPOOLDIR_H */ diff --git a/libmy/varint.c b/libmy/varint.c deleted file mode 100644 index 69692747..00000000 --- a/libmy/varint.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2012, 2013 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -/* - * Copyright (c) 2008, Dave Benson. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "varint.h" - -unsigned -varint_length(uint64_t v) -{ - unsigned len = 1; - while (v >= 128) { - v >>= 7; - len++; - } - return (len); -} - -unsigned -varint_length_packed(const uint8_t *data, size_t len_data) -{ - unsigned i = 0; - size_t len = len_data; - while (len--) { - if ((data[i] & 0x80) == 0) - break; - i++; - } - if (i == len_data) - return (0); - return (i + 1); -} - -size_t -varint_encode32(uint8_t *src_ptr, uint32_t v) -{ - static const unsigned B = 128; - uint8_t *ptr = src_ptr; - if (v < (1 << 7)) { - *(ptr++) = v; - } else if (v < (1 << 14)) { - *(ptr++) = v | B; - *(ptr++) = v >> 7; - } else if (v < (1 << 21)) { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = v >> 14; - } else if (v < (1 << 28)) { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = (v >> 14) | B; - *(ptr++) = v >> 21; - } else { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = (v >> 14) | B; - *(ptr++) = (v >> 21) | B; - *(ptr++) = v >> 28; - } - return ((size_t) (ptr - src_ptr)); -} - -size_t -varint_encode64(uint8_t *src_ptr, uint64_t v) -{ - static const unsigned B = 128; - uint8_t *ptr = src_ptr; - while (v >= B) { - *(ptr++) = (v & (B - 1)) | B; - v >>= 7; - } - *(ptr++) = (uint8_t) v; - return ((size_t) (ptr - src_ptr)); -} - -size_t -varint_decode32(const uint8_t *data, uint32_t *value) -{ - unsigned len = varint_length_packed(data, 5); - uint32_t val = data[0] & 0x7f; - if (len > 1) { - val |= ((data[1] & 0x7f) << 7); - if (len > 2) { - val |= ((data[2] & 0x7f) << 14); - if (len > 3) { - val |= ((data[3] & 0x7f) << 21); - if (len > 4) - val |= (data[4] << 28); - } - } - } - *value = val; - return ((size_t) len); -} - -size_t -varint_decode64(const uint8_t *data, uint64_t *value) -{ - unsigned shift, i; - unsigned len = varint_length_packed(data, 10); - uint64_t val; - if (len < 5) { - size_t tmp_len; - uint32_t tmp; - tmp_len = varint_decode32(data, &tmp); - *value = tmp; - return (tmp_len); - } - val = ((data[0] & 0x7f)) - | ((data[1] & 0x7f) << 7) - | ((data[2] & 0x7f) << 14) - | ((data[3] & 0x7f) << 21); - shift = 28; - for (i = 4; i < len; i++) { - val |= (((uint64_t)(data[i] & 0x7f)) << shift); - shift += 7; - } - *value = val; - return ((size_t) len); -} diff --git a/libmy/varint.h b/libmy/varint.h deleted file mode 100644 index ad91b0cc..00000000 --- a/libmy/varint.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MY_VARINT_H -#define MY_VARINT_H - -#include -#include - -unsigned varint_length(uint64_t v); -unsigned varint_length_packed(const uint8_t *buf, size_t len_buf); -size_t varint_encode32(uint8_t *ptr, uint32_t value); -size_t varint_encode64(uint8_t *ptr, uint64_t value); -size_t varint_decode32(const uint8_t *ptr, uint32_t *value); -size_t varint_decode64(const uint8_t *ptr, uint64_t *value); - -#endif /* MY_VARINT_H */ diff --git a/libmy/zonefile.c b/libmy/zonefile.c deleted file mode 100644 index eddaa8b1..00000000 --- a/libmy/zonefile.c +++ /dev/null @@ -1,188 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "my_alloc.h" -#include "ubuf.h" -#include "zonefile.h" - -struct zonefile { - FILE *fp; - bool eof; - bool is_pipe; - bool valid; - ldns_rdf *domain; - ldns_rdf *origin; - ldns_rdf *prev; - ldns_rr *rr_soa; - uint32_t ttl; - size_t count; -}; - -static ldns_status -read_soa(struct zonefile *z) -{ - ldns_rr *rr; - ldns_status status; - - for (;;) { - status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); - switch (status) { - case LDNS_STATUS_OK: - goto out; - case LDNS_STATUS_SYNTAX_EMPTY: - case LDNS_STATUS_SYNTAX_TTL: - case LDNS_STATUS_SYNTAX_ORIGIN: - status = LDNS_STATUS_OK; - break; - default: - goto out; - } - } -out: - if (status != LDNS_STATUS_OK) { - z->valid = false; - return (LDNS_STATUS_ERR); - } - - if (ldns_rr_get_type(rr) != LDNS_RR_TYPE_SOA) { - ldns_rr_free(rr); - z->valid = false; - return (LDNS_STATUS_ERR); - } - - z->count = 1; - z->domain = ldns_rdf_clone(ldns_rr_owner(rr)); - z->origin = ldns_rdf_clone(ldns_rr_owner(rr)); - z->rr_soa = rr; - return (LDNS_STATUS_OK); -} - -struct zonefile * -zonefile_init_fname(const char *fname) -{ - struct zonefile *z = my_calloc(1, sizeof(struct zonefile)); - - size_t len_fname = strlen(fname); - if (len_fname >= 3 && - fname[len_fname - 3] == '.' && - fname[len_fname - 2] == 'g' && - fname[len_fname - 1] == 'z') - { - ubuf *u = ubuf_new(); - ubuf_add_cstr(u, "zcat "); - ubuf_add_cstr(u, fname); - z->fp = popen(ubuf_cstr(u), "r"); - z->is_pipe = true; - ubuf_destroy(&u); - } else { - z->fp = fopen(fname, "r"); - } - - if (z->fp == NULL) - return (NULL); - - z->valid = true; - if (read_soa(z) != LDNS_STATUS_OK) - zonefile_destroy(&z); - - return (z); -} - -void -zonefile_destroy(struct zonefile **z) -{ - if (*z) { - if ((*z)->fp) { - if ((*z)->is_pipe) - pclose((*z)->fp); - else - fclose((*z)->fp); - } - if ((*z)->origin) - ldns_rdf_deep_free((*z)->origin); - if ((*z)->prev) - ldns_rdf_deep_free((*z)->prev); - if ((*z)->domain) - ldns_rdf_deep_free((*z)->domain); - if ((*z)->rr_soa) - ldns_rr_free((*z)->rr_soa); - free(*z); - *z = NULL; - } -} - -const ldns_rdf * -zonefile_get_domain(struct zonefile *z) -{ - return (z->domain); -} - -size_t -zonefile_get_count(struct zonefile *z) -{ - return (z->count); -} - -uint32_t -zonefile_get_serial(struct zonefile *z) -{ - ldns_rdf *rdf = ldns_rr_rdf(z->rr_soa, 2); - assert(rdf != NULL); - return (ldns_rdf2native_int32(rdf)); -} - -ldns_status -zonefile_read(struct zonefile *z, ldns_rr **out) -{ - ldns_rr *rr; - ldns_status status = LDNS_STATUS_OK; - - if (z->eof) { - *out = NULL; - return (LDNS_STATUS_OK); - } - - if (!z->valid) - return (LDNS_STATUS_ERR); - - if (z->count == 1 && z->rr_soa != NULL) { - *out = z->rr_soa; - z->rr_soa = NULL; - return (LDNS_STATUS_OK); - } - for (;;) { - if (feof(z->fp)) { - *out = NULL; - z->eof = true; - return (LDNS_STATUS_OK); - } - status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); - switch (status) { - case LDNS_STATUS_OK: - if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_SOA) { - ldns_rr_free(rr); - *out = NULL; - return (LDNS_STATUS_OK); - } - z->count++; - goto out; - case LDNS_STATUS_SYNTAX_EMPTY: - case LDNS_STATUS_SYNTAX_TTL: - case LDNS_STATUS_SYNTAX_ORIGIN: - status = LDNS_STATUS_OK; - break; - default: - goto out; - } - } -out: - if (status != LDNS_STATUS_OK) - return (status); - *out = rr; - return (status); -} diff --git a/libmy/zonefile.h b/libmy/zonefile.h deleted file mode 100644 index f6d78562..00000000 --- a/libmy/zonefile.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef MY_ZONEFILE_H -#define MY_ZONEFILE_H - -#include - -struct zonefile; - -struct zonefile * -zonefile_init_fname(const char *fname); - -void -zonefile_destroy(struct zonefile **); - -const ldns_rdf * -zonefile_get_domain(struct zonefile *); - -size_t -zonefile_get_count(struct zonefile *); - -uint32_t -zonefile_get_serial(struct zonefile *); - -ldns_status -zonefile_read(struct zonefile *, ldns_rr **); - -#endif /* MY_ZONEFILE_H */ From 6740971987914ad8c2198c771e69f0dc81f69e6b Mon Sep 17 00:00:00 2001 From: "Jeremy C. Reed" Date: Thu, 4 Jun 2026 16:27:00 +0000 Subject: [PATCH 08/18] fix closing docbook tag it had a parser error with opening and ending tag mismatch for replaceable --- doc/docbook/nmsgtool.docbook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 96304bf2..8d4dd1c5 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -172,7 +172,7 @@ pidfile Write nmsgtool's process ID into a file - identified by pidfile. When + identified by pidfile. When nmsgtool exits, this file's contents will be erased. From 037cb969bdee7ef877c69d2724ea755cbb7e47c0 Mon Sep 17 00:00:00 2001 From: "Jeremy C. Reed" Date: Tue, 9 Jun 2026 20:25:48 +0000 Subject: [PATCH 09/18] fix docbook for end of sentence period outside of para (roff was seen in rendered manpage) --- doc/docbook/nmsgtool.docbook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 8d4dd1c5..58b49191 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -114,7 +114,7 @@ verbose and is very verbose. If the NMSG_KAFKA_LOG_LEVEL environment variable is set and a Kafka input/output is used, Kafka will - log non-errors at the specified logging level. + log non-errors at the specified logging level. From 72e0803ef37a45c1661a62684efe2a0f1b054b67 Mon Sep 17 00:00:00 2001 From: Stephen Watt Date: Mon, 22 Jun 2026 16:53:20 -0400 Subject: [PATCH 10/18] Add support for single nmsg payload kafka input/output sources via nmsgp: proto. --- debian/libnmsg8.symbols | 2 + doc/docbook/nmsgtool.1 | 130 ++++++++++++++++++++++++++++++----- doc/docbook/nmsgtool.docbook | 70 +++++++++++++------ nmsg/input.c | 36 ++++++++++ nmsg/input.h | 22 +++++- nmsg/input_nmsg.c | 47 +++++++++++++ nmsg/output.c | 50 +++++++++++++- nmsg/output.h | 16 +++++ nmsg/output_nmsg.c | 46 +++++++++++++ nmsg/private.h | 5 ++ src/io.c | 81 +++++++++++++++++++++- src/nmsgtool.c | 2 +- tests/test-private.c | 33 ++++++++- 13 files changed, 492 insertions(+), 48 deletions(-) diff --git a/debian/libnmsg8.symbols b/debian/libnmsg8.symbols index 7c4e9139..33dd3b5a 100644 --- a/debian/libnmsg8.symbols +++ b/debian/libnmsg8.symbols @@ -30,6 +30,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_input_open_json@Base 0.10.0 nmsg_input_open_kafka_endpoint@Base 1.2.0 nmsg_input_open_kafka_json@Base 1.2.0 + nmsg_input_open_kafka_payload@Base 1.4.0 nmsg_input_open_null@Base 0.7.0 nmsg_input_open_pcap@Base 0.5.0 nmsg_input_open_pres@Base 0.5.0 @@ -138,6 +139,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_open_sock@Base 0.5.0 nmsg_output_open_zmq@Base 0.14.0 nmsg_output_open_zmq_endpoint@Base 0.14.0 + nmsg_output_open_kafka_payload@Base 1.4.0 nmsg_output_set_buffered@Base 0.5.0 nmsg_output_set_endline@Base 0.5.0 nmsg_output_set_filter_msgtype@Base 0.5.0 diff --git a/doc/docbook/nmsgtool.1 b/doc/docbook/nmsgtool.1 index 7a1633e5..3d398c26 100644 --- a/doc/docbook/nmsgtool.1 +++ b/doc/docbook/nmsgtool.1 @@ -2,12 +2,12 @@ .\" Title: nmsgtool .\" Author: [FIXME: author] [see http://www.docbook.org/tdg5/en/html/author] .\" Generator: DocBook XSL Stylesheets v1.79.2 -.\" Date: 10/02/2024 +.\" Date: 04/22/2026 .\" Manual: .\" Source: .\" Language: English .\" -.TH "NMSGTOOL" "1" "10/02/2024" "" "" +.TH "NMSGTOOL" "1" "04/22/2026" "" "" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -118,6 +118,8 @@ Writing ASCII presentation form data to a file\&. See the option\&. .RE .PP +There are more input and output types supported, depending upon the installation\&. See below in the options list\&. +.PP Reading or writing data in a non\-NMSG format requires the use of an external module (called an "nmsgpb module") to convert to or from NMSG format\&. \fBnmsgtool\fR selects an nmsgpb module based on a vendor ID and message type\&. For input data, these fields must be set with the @@ -142,15 +144,15 @@ Increment debugging level\&. \fB\-dd\fR is verbose and \fB\-dddd\fR -is very verbose\&. -If the +is very verbose\&. If the \fBNMSG_KAFKA_LOG_LEVEL\fR -environment variable is set and a Kafka input/output is used, Kafka will log non-errors at the specified logging level\&. +environment variable is set and a Kafka input/output is used, Kafka will log non\-errors at the specified logging level\&. .RE .PP -\fB\-v\fR \fIversion\fR +\fB\-v\fR, \fB\-\-version\fR .RS 4 -Print the version number of \fBnmsgtool\fR. +Print +\fBnmsgtool\fR\*(Aqs version number\&. .RE .PP \fB\-V\fR \fIvendor\fR, \fB\-\-vendor\fR \fIvendor\fR @@ -167,24 +169,34 @@ Set the message type field of generated NMSG payloads to the message type identi .PP \fB\-B\fR \fIbyterate\fR, \fB\-\-byterate\fR \fIbyterate\fR .RS 4 -Set the maximum bytes-per-second that libnmsg will process from file inputs. +Set the maximum bytes\-per\-second that libnmsg will process from file inputs\&. .RE .PP -\fB\-D\fR \fIdaemon\fR +\fB\-D\fR, \fB\-\-daemon\fR .RS 4 -Fork \fBnmsgtool\fR into the background as a daemon process. +Fork +\fBnmsgtool\fR +into the background as a daemon process\&. .RE .PP \fB\-P\fR \fIpidfile\fR, \fB\-\-pidfile\fR \fIpidfile\fR .RS 4 -Write \fBnmsgtool\fR's process ID into a file identified by \fIpidfile\fR. When -\fBnmsgtool\fR exits, this file's contents will be erased. +Write +\fBnmsgtool\fR\*(Aqs process ID into a file identified by +\fIpidfile\fR\&. When +\fB nmsgtool\fR +exits, this file\*(Aqs contents will be erased\&. .RE .PP \fB\-U\fR \fIusername\fR, \fB\-\-username\fR \fIusername\fR .RS 4 -Attempt to drop root privileges and run \fBnmsgtool\fR as user \fIusername\fR. -If the current user is \fIusername\fR, \fBnmsgtool\fR will exit. +Attempt to drop root privileges and run +\fBnmsgtool \fR +as user +\fIusername\fR\&. If the current user is +\fIusername\fR, +\fB nmsgtool\fR +will exit\&. .RE .PP \fB\-e\fR \fIendline\fR, \fB\-\-endline\fR \fIendline\fR @@ -264,9 +276,48 @@ environment variable\&. Read NMSG payloads in either binary or JSON format from a Kafka endpoint\&. The address \fIkafka\fR has format -\fBproto:topic[#partition|%group_id]@broker[:port][,offset]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. Also optional is an offset consisting of either a numerical value or the string \*(Aqoldest\*(Aq or \*(Aqnewest\*(Aq in order to start retrieval at the oldest/newest messages in the Kafka topic\&. An example of a possible +\fBproto:topic[#partition|%group_id]@broker[:port][,offset]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. Also optional is an offset consisting of either a numerical value or the string \*(Aqoldest\*(Aq or \*(Aqnewest\*(Aq in order to start retrieval at the oldest/newest messages in the Kafka topic\&. +.sp +There are three \fIkafka\fR -endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&. Configuration for Kafka can be supplied through the +endpoint proto types defined: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +json: to read json payloads from each kafka record +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsg: to read a full binary NMSG container from each kafka record +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsgp: to read a single binary NMSG payload from each kafka record +.RE +.sp +Example: "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&. +.sp +Configuration for Kafka can be supplied through the \fBNMSG_KAFKA_CONFIG\fR environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&. .RE @@ -503,9 +554,50 @@ Write NMSG payloads to a file\&. Write NMSG payloads in either binary or JSON format to a Kafka endpoint\&. The address \fIkafka\fR has format -\fBproto:topic[#partition|%group_id]@broker[:port]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. An example of a possible +\fBproto:topic[#partition|%group_id]@broker[:port]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. +.sp +There are three \fIkafka\fR -endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. Note that nmsgtool ignores offsets for Kafka producers\&. Configuration for Kafka can be supplied through the +endpoint proto types defined: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +json: to write JSON payloads to Kafka\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsg: to write a full binary NMSG container in each kafka record\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsgp: to write a single binary NMSG payload in each kafka record\&. +.RE +.sp +Example: "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. +.sp +Note that nmsgtool ignores offsets for Kafka producers\&. +.sp +Configuration for Kafka can be supplied through the \fBNMSG_KAFKA_CONFIG\fR environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&. .RE @@ -789,7 +881,7 @@ To read NMSG payloads from a file and write them to a ZeroMQ "PUSH" socket over .PP This attempts to connect to a TCP reader on 127\&.0\&.0\&.1:5555, such as the nmsgtool command in the previous example\&. .PP -To read NMSG payloads from an SIE channel named "ch222" and write them to stdout while writing IO stats every 3 seconds to a file name "ch222.stats": +To read NMSG payloads from an SIE channel named "ch222" and write them to stdout while writing IO stats every 3 seconds to a file name "ch222\&.stats": .sp .if n \{\ .RS 4 diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 58b49191..021afef6 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -76,6 +76,9 @@ + There are more input and output types supported, depending upon the installation. See below in the + options list. + Reading or writing data in a non-NMSG format requires the use of an external module (called an "nmsgpb module") to convert to or from NMSG format. nmsgtool selects an @@ -125,7 +128,7 @@ Print nmsgtool's version number. - + vendor vendor @@ -153,7 +156,7 @@ byterate byterate - Set the maximum bytes-per-second that libnmsg will + Set the maximum bytes-per-second that libnmsg will process from file inputs. @@ -162,7 +165,7 @@ - Fork nmsgtool into the background as a daemon + Fork nmsgtool into the background as a daemon process. @@ -171,23 +174,23 @@ pidfile pidfile - Write nmsgtool's process ID into a file + Write nmsgtool's process ID into a file identified by pidfile. When nmsgtool exits, this file's contents will be erased. - + username username Attempt to drop root privileges and run nmsgtool - as user username. If the + as user username. If the current user is username, nmsgtool will exit. - + endline endline @@ -293,13 +296,26 @@ Also optional is an offset consisting of either a numerical value or the string 'oldest' or 'newest' in order to start retrieval at the oldest/newest messages in the Kafka topic. - An example of a possible kafka endpoint is - "nmsg:ch202#0@kafka.example.com:9092,3000" to indicate that nmsgtool shall read nmsg - containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at - kafka.example.com, port 9092. - Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. - This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. + + There are three kafka endpoint proto types + defined: + + + + json: to read json payloads from each kafka record + + + nmsg: to read a full binary NMSG container from each kafka record + + + nmsgp: to read a single binary NMSG payload from each kafka record + + + Example: "nmsg:ch202#0@kafka.example.com:9092,3000" to indicate that nmsgtool shall read nmsg containers + from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka.example.com, port 9092. + Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. + This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. @@ -534,13 +550,27 @@ Write NMSG payloads in either binary or JSON format to a Kafka endpoint. The address kafka has format proto:topic[#partition|%group_id]@broker[:port]. Either a partition number or a consumer group ID may be optionally supplied. - An example of a possible kafka endpoint is - "nmsg:ch202#0@kafka.example.com:9092" to indicate that nmsgtool shall write - nmsg containers to topic "ch202" on partition 0 to Kafka - broker kafka.example.com, port 9092. - Note that nmsgtool ignores offsets for Kafka producers. - Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. - This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. + + + There are three kafka endpoint proto types + defined: + + + + json: to write JSON payloads to Kafka. + + + nmsg: to write a full binary NMSG container in each kafka record. + + + nmsgp: to write a single binary NMSG payload in each kafka record. + + + Example: "nmsg:ch202#0@kafka.example.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on + partition 0 to Kafka broker kafka.example.com, port 9092. + Note that nmsgtool ignores offsets for Kafka producers. + Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. + This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. diff --git a/nmsg/input.c b/nmsg/input.c index 9debc7a5..ac32c5d8 100644 --- a/nmsg/input.c +++ b/nmsg/input.c @@ -73,6 +73,41 @@ nmsg_input_open_kafka_json(const char *address __attribute__((unused))) { } #endif /* (defined HAVE_LIBRDKAFKA) */ +#ifdef HAVE_LIBRDKAFKA +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address) +{ + struct nmsg_input *input; + + input = calloc(1, sizeof(*input)); + if (input == NULL) + return (NULL); + + input->kafka = calloc(1, sizeof(*(input->kafka))); + if (input->kafka == NULL) { + free(input); + return (NULL); + } + + input->type = nmsg_input_type_kafka_payload; + input->read_fp = _input_kafka_payload_read; + + input->kafka->ctx = kafka_create_consumer(address, NMSG_RBUF_TIMEOUT); + if (input->kafka->ctx == NULL) { + free(input->kafka); + free(input); + return (NULL); + } + + return (input); +} +#else /* HAVE_LIBRDKAFKA */ +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address __attribute__((unused))) { + return (NULL); +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBRDKAFKA nmsg_input_t _input_open_kafka(void *s) { @@ -288,6 +323,7 @@ nmsg_input_close(nmsg_input_t *input) { free((*input)->json); break; case nmsg_input_type_kafka_json: + case nmsg_input_type_kafka_payload: #ifdef HAVE_LIBRDKAFKA kafka_ctx_destroy(&(*input)->kafka->ctx); free((*input)->kafka); diff --git a/nmsg/input.h b/nmsg/input.h index afefe261..35afda28 100644 --- a/nmsg/input.h +++ b/nmsg/input.h @@ -59,6 +59,7 @@ typedef enum { nmsg_input_type_callback, nmsg_input_type_json, /*%< JSON form */ nmsg_input_type_kafka_json, /*%< NMSG payloads from Kafka in JSON form */ + nmsg_input_type_kafka_payload, /*%< NMSG single binary payloads from Kafka */ } nmsg_input_type; /** @@ -133,9 +134,9 @@ nmsg_input_open_zmq_endpoint(void *zmq_ctx, const char *ep); * Only if a partition number has been specified can offset be a numeric value. * Note that only new consumer group IDs will honor these directives. * - * The value of proto must be either "nmsg" (binary container input) or "json" - * (JSON-serialized payloads) and either or both a partition number and offset - * value may be optionally supplied. + * The value of proto must be "nmsg" (binary container input), "nmsgp" (single + * payload input), or "json" (JSON-serialized payloads) and either or both a + * partition number and offset value may be optionally supplied. * * \see nmsg_output_open_kafka_endpoint() * @@ -214,6 +215,21 @@ nmsg_input_open_json(int fd); nmsg_input_t nmsg_input_open_kafka_json(const char *address); +/** + * Initialize a new NMSG single payload input from a Kafka broker. + * + * Each Kafka message is expected to contain a single serialized + * NmsgPayload protobuf (not a full NMSG container). + * + * See nmsg_input_open_kafka_endpoint for the details of the address string. + * + * \param[in] address Kafka endpoint address string. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address); + /** * Initialize a new NMSG pcap input from a pcap descriptor. diff --git a/nmsg/input_nmsg.c b/nmsg/input_nmsg.c index 5d2d2774..08b8592c 100644 --- a/nmsg/input_nmsg.c +++ b/nmsg/input_nmsg.c @@ -416,6 +416,53 @@ _input_nmsg_read_container_kafka(nmsg_input_t input, Nmsg__Nmsg **nmsg) { } #endif /* HAVE_LIBRDKAFKA */ +#ifdef HAVE_LIBRDKAFKA +nmsg_res +_input_kafka_payload_read(nmsg_input_t input, nmsg_message_t *msg) { + nmsg_res res; + uint8_t *buf; + size_t buf_len; + Nmsg__NmsgPayload *np; + + res = kafka_read_start(input->kafka->ctx, &buf, &buf_len); + if (res != nmsg_res_success) { + kafka_read_finish(input->kafka->ctx); + return res; + } + + if (buf_len == 0) { + kafka_read_finish(input->kafka->ctx); + return nmsg_res_failure; + } + + np = nmsg__nmsg_payload__unpack(NULL, buf_len, buf); + + kafka_read_finish(input->kafka->ctx); + + if (np == NULL) { + _nmsg_dprintf(1, "%s: failed to unpack payload\n", __func__); + return nmsg_res_parse_error; + } + + /* filter (vid, msgtype) */ + if (input->do_filter && + (input->filter_vid != np->vid || + input->filter_msgtype != np->msgtype)) + { + _nmsg_payload_free(&np); + return nmsg_res_again; + } + + *msg = _nmsg_message_from_payload(np); + if (*msg == NULL) { + _nmsg_payload_free(&np); + return nmsg_res_memfail; + } + + return nmsg_res_success; +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBZMQ nmsg_res _input_nmsg_read_container_zmq(nmsg_input_t input, Nmsg__Nmsg **nmsg) { diff --git a/nmsg/output.c b/nmsg/output.c index cf3e0d91..8813c6bc 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -85,6 +85,47 @@ nmsg_output_open_kafka_json(const char *addr __attribute__((unused)), } #endif /* HAVE_LIBRDKAFKA */ +#ifdef HAVE_LIBRDKAFKA +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr, const char *key_field) +{ + struct nmsg_output *output; + + output = calloc(1, sizeof(*output)); + if (output == NULL) + return (NULL); + + output->kafka = calloc(1, sizeof(*(output->kafka))); + if (output->kafka == NULL) { + free(output); + return (NULL); + } + + output->type = nmsg_output_type_kafka_payload; + output->write_fp = _output_kafka_payload_write; + output->flush_fp = _output_kafka_payload_flush; + + output->kafka->ctx = kafka_create_producer(addr, NMSG_RBUF_TIMEOUT); + if (!output->kafka->ctx) { + free(output->kafka); + free(output); + return (NULL); + } + + if (key_field != NULL) + output->kafka->key_field = strdup(key_field); + + return output; +} +#else /* HAVE_LIBRDKAFKA */ +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr __attribute__((unused)), + const char *key_field __attribute__((unused))) +{ + return (NULL); +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBRDKAFKA nmsg_output_t _output_open_kafka(void *s, size_t bufsz) { @@ -282,6 +323,7 @@ nmsg_output_close(nmsg_output_t *output) { free((*output)->json); break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: #ifdef HAVE_LIBRDKAFKA kafka_ctx_destroy(&(*output)->kafka->ctx); if ((*output)->kafka->key_field != NULL) @@ -289,6 +331,7 @@ nmsg_output_close(nmsg_output_t *output) { free((*output)->kafka); #else /* HAVE_LIBRDKAFKA */ assert((*output)->type != nmsg_output_type_kafka_json); + assert((*output)->type != nmsg_output_type_kafka_payload); #endif /* HAVE_LIBRDKAFKA */ break; case nmsg_output_type_callback: @@ -390,6 +433,7 @@ nmsg_output_set_source(nmsg_output_t output, unsigned source) { output->json->source = source; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->source = source; default: break; @@ -409,6 +453,7 @@ nmsg_output_set_operator(nmsg_output_t output, unsigned operator) { output->json->operator = operator; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->operator = operator; default: break; @@ -428,6 +473,7 @@ nmsg_output_set_group(nmsg_output_t output, unsigned group) { output->json->group = group; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->group = group; default: break; @@ -440,7 +486,9 @@ _output_stop(nmsg_output_t output) { #ifdef HAVE_LIBRDKAFKA if (output->type == nmsg_output_type_kafka_json) kafka_stop(output->kafka->ctx); - if (output->type == nmsg_output_type_stream && + if (output->type == nmsg_output_type_kafka_payload) + kafka_stop(output->kafka->ctx); + else if (output->type == nmsg_output_type_stream && output->stream != NULL && output->stream->type == nmsg_stream_type_kafka) kafka_stop(output->stream->kafka); diff --git a/nmsg/output.h b/nmsg/output.h index 83a8048f..f09ba6c1 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -42,6 +42,7 @@ typedef enum { nmsg_output_type_callback, nmsg_output_type_json, nmsg_output_type_kafka_json, + nmsg_output_type_kafka_payload, } nmsg_output_type; /** @@ -193,6 +194,21 @@ nmsg_output_open_json(int fd); nmsg_output_t nmsg_output_open_kafka_json(const char *addr, const char *key_field); +/** + * Initialize a new NMSG binary-payload output to a Kafka broker. + * + * Each Kafka message carries a single serialized binary NMSG Payload protobuf. + * See nmsg_input_open_kafka_endpoint for the details of the address string. + * + * \param[in] addr Kafka endpoint address string (without proto: prefix). + * \param[in] key_field An optional NMSG field name whose content will be + * used as a Kafka producer key. Otherwise, its value should be NULL. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr, const char *key_field); + /** * Initialize a new nmsg output closure. This allows a user-provided callback to * function as an nmsg output, for instance to participate in an nmsg_io loop. diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index f97c7b85..13421e9b 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -293,6 +293,52 @@ send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) return (res); } +#ifdef HAVE_LIBRDKAFKA +nmsg_res +_output_kafka_payload_write(nmsg_output_t output, nmsg_message_t msg) { + nmsg_res res; + struct nmsg_strbuf_storage key_sbs; + struct nmsg_strbuf *key_sb = NULL; + uint8_t *buf = NULL, *key = NULL; + size_t buf_len, key_len = 0; + + assert(msg->np != NULL); + + buf_len = nmsg__nmsg_payload__get_packed_size(msg->np); + buf = malloc(buf_len); + if (buf == NULL) + return nmsg_res_memfail; + nmsg__nmsg_payload__pack(msg->np, buf); + + if (output->kafka->key_field != NULL) { + key_sb = _nmsg_strbuf_init(&key_sbs); + res = _nmsg_message_get_field_value_as_key(msg, output->kafka->key_field, key_sb); + if (res != nmsg_res_success) + goto out; + + key_len = nmsg_strbuf_len(key_sb); + key = (uint8_t *) key_sb->data; + } + + /* kafka_write() takes ownership of buf */ + res = kafka_write(output->kafka->ctx, key, key_len, buf, buf_len); + buf = NULL; + +out: + if (buf != NULL) + free(buf); + if (key_sb != NULL) + _nmsg_strbuf_destroy(&key_sbs); + return res; +} + +nmsg_res +_output_kafka_payload_flush(nmsg_output_t output) { + kafka_flush(output->kafka->ctx); + return nmsg_res_success; +} +#endif /* HAVE_LIBRDKAFKA */ + static void header_serialize(uint8_t *buf, uint8_t flags, uint32_t len) { diff --git a/nmsg/private.h b/nmsg/private.h index 125752a8..42caf21e 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -543,6 +543,7 @@ nmsg_res _input_nmsg_read_container_file(nmsg_input_t, Nmsg__Nmsg **); nmsg_res _input_nmsg_read_container_sock(nmsg_input_t, Nmsg__Nmsg **); #ifdef HAVE_LIBRDKAFKA nmsg_res _input_nmsg_read_container_kafka(nmsg_input_t, Nmsg__Nmsg **); +nmsg_res _input_kafka_payload_read(nmsg_input_t, nmsg_message_t *); #endif /* HAVE_LIBRDKAFKA */ #ifdef HAVE_LIBZMQ nmsg_res _input_nmsg_read_container_zmq(nmsg_input_t, Nmsg__Nmsg **); @@ -585,6 +586,10 @@ nmsg_output_t _output_open_kafka(void *s, size_t bufsz); /* from output_nmsg.c */ nmsg_res _output_nmsg_flush(nmsg_output_t); nmsg_res _output_nmsg_write(nmsg_output_t, nmsg_message_t); +#ifdef HAVE_LIBRDKAFKA +nmsg_res _output_kafka_payload_write(nmsg_output_t, nmsg_message_t); +nmsg_res _output_kafka_payload_flush(nmsg_output_t); +#endif /* HAVE_LIBRDKAFKA */ /* from output_pres.c */ nmsg_res _output_pres_write(nmsg_output_t, nmsg_message_t); diff --git a/src/io.c b/src/io.c index d5cd8065..4a602969 100644 --- a/src/io.c +++ b/src/io.c @@ -279,6 +279,73 @@ _add_kafka_json_output(nmsgtool_ctx *c __attribute__((unused)), } #endif /* HAVE_LIBRDKAFKA */ +#ifdef HAVE_LIBRDKAFKA +static void +_add_kafka_payload_output(nmsgtool_ctx *c, const char *str_address) { + nmsg_res res; + nmsg_output_t output; + + output = nmsg_output_open_kafka_payload(str_address, c->kafka_key_field); + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg Kafka nmsgp output: %s\n", argv_program, str_address); + if (output == NULL) { + fprintf(stderr, "%s: nmsg_output_open_kafka_payload() failed\n", argv_program); + exit(1); + } + setup_nmsg_output(c, output); + if (c->kicker != NULL) + res = nmsg_io_add_output(c->io, output, (void *) -1); + else + res = nmsg_io_add_output(c->io, output, NULL); + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_output() failed\n", argv_program); + exit(1); + } + c->n_outputs += 1; +} +#else /* HAVE_LIBRDKAFKA */ +static void +_add_kafka_payload_output(nmsgtool_ctx *c __attribute__((unused)), + const char *str_address __attribute__((unused))) +{ + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_LIBRDKAFKA */ + +#ifdef HAVE_LIBRDKAFKA +static void +_add_kafka_payload_input(nmsgtool_ctx *c, const char *str_address) { + nmsg_res res; + nmsg_input_t input; + + input = nmsg_input_open_kafka_payload(str_address); + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg Kafka nmsgp input: %s\n", argv_program, str_address); + if (input == NULL) { + fprintf(stderr, "%s: nmsg_input_open_kafka_payload() failed\n", argv_program); + exit(1); + } + setup_nmsg_input(c, input); + res = nmsg_io_add_input(c->io, input, NULL); + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_input() failed\n", argv_program); + exit(1); + } + c->n_inputs += 1; +} +#else /* HAVE_LIBRDKAFKA */ +static void +_add_kafka_payload_input(nmsgtool_ctx *c __attribute__((unused)), + const char *str_address __attribute__((unused))) +{ + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBRDKAFKA static void _add_kafka_nmsg_input(nmsgtool_ctx *c, const char *str_address) { @@ -351,12 +418,17 @@ add_kafka_input(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_input(c, addr); return; } + addr = _strip_prefix_if_exists(str_address, "nmsgp:"); + if (addr != NULL) { + _add_kafka_payload_input(c, addr); + return; + } addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_input(c, addr); return; } - fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", + fprintf(stderr, "%s: Error: nmsg, nmsgp, or json protocol must be set for Kafka endpoint\n", argv_program); exit(EXIT_FAILURE); } @@ -368,12 +440,17 @@ add_kafka_output(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_output(c, addr); return; } + addr = _strip_prefix_if_exists(str_address, "nmsgp:"); + if (addr != NULL) { + _add_kafka_payload_output(c, addr); + return; + } addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_output(c, addr); return; } - fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", + fprintf(stderr, "%s: Error: nmsg, nmsgp, or json protocol must be set for Kafka endpoint\n", argv_program); exit(EXIT_FAILURE); } diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 62a9c506..3d870345 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -148,7 +148,7 @@ static argv_t args[] = { &ctx.r_kafka, "kafka", #ifdef HAVE_LIBRDKAFKA - "read nmsg data from Kafka (binary or json)" }, + "read nmsg data from Kafka (nmsg, nmsgp, or json)" }, #else /* HAVE_LIBRDKAFKA */ "read nmsg data from Kafka topic (no support)" }, #endif /* HAVE_LIBRDKAFKA */ diff --git a/tests/test-private.c b/tests/test-private.c index 788e3548..ed75641f 100644 --- a/tests/test-private.c +++ b/tests/test-private.c @@ -178,7 +178,35 @@ test_kafka_key(void) { l_return_test_status(); } -#endif /* (defined HAVE_LIBRDKAFKA) */ + +/* Test null/invalid argument handling for nmsg_output_open_kafka_payload nmsg_input_open_kafka_payload. */ +static int +test_kafka_payload_papi(void) +{ + nmsg_output_t o; + + /* NULL address must return NULL without crashing. */ + o = nmsg_output_open_kafka_payload(NULL, NULL); + check(o == NULL); + + /* Address with no '@' is structurally invalid and must return NULL. */ + o = nmsg_output_open_kafka_payload("topic-no-broker", NULL); + check(o == NULL); + + nmsg_input_t i; + + /* NULL address must return NULL without crashing. */ + i = nmsg_input_open_kafka_payload(NULL); + check(i == NULL); + + /* Address with no '@' is structurally invalid and must return NULL. */ + i = nmsg_input_open_kafka_payload("topic-no-broker"); + check(i == NULL); + + l_return_test_status(); +} + +#endif /* HAVE_LIBRDKAFKA */ static int _test_config_file_papi_null(void) { @@ -340,7 +368,8 @@ main(void) #if (defined HAVE_LIBRDKAFKA) check_explicit2_display_only(test_kafka_papi() == 0, "test-private / test_kafka_papi"); check_explicit2_display_only(test_kafka_key() == 0, "test-private / test_kafka_key"); -#endif /* (defined HAVE_LIBRDKAFKA) */ + check_explicit2_display_only(test_kafka_payload_papi() == 0, "test-private / test_kafka_payload_papi"); +#endif /* HAVE_LIBRDKAFKA */ g_check_test_status(false); } From 188ef14286c64e48b24acac186b50186748dc562 Mon Sep 17 00:00:00 2001 From: Chris Mikkelson Date: Tue, 30 Jun 2026 09:24:40 -0500 Subject: [PATCH 11/18] Handle fragmented containers in zmq, kafka inputs Properly propagate the `nmsg_res_again` return of _input_nmsg_unpack_container through _input_process_buffer_into_container. --- nmsg/input_nmsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmsg/input_nmsg.c b/nmsg/input_nmsg.c index 08b8592c..839098ee 100644 --- a/nmsg/input_nmsg.c +++ b/nmsg/input_nmsg.c @@ -390,7 +390,7 @@ _input_process_buffer_into_container(nmsg_input_t input, Nmsg__Nmsg **nmsg, uint /* expire old outstanding fragments */ _input_frag_gc(input->stream); - return nmsg_res_success; + return (res); } #endif /* defined(HAVE_LIBRDKAFKA) || defined(HAVE_LIBZMQ) */ From c1cf638ba841bfccd7828fa171158c5ed0f66e9c Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Thu, 20 Aug 2026 13:38:07 -0400 Subject: [PATCH 12/18] Add --zasync: compress nmsg file containers on a background thread, one in flight --- nmsg/output.c | 43 ++++++- nmsg/output.h | 23 ++++ nmsg/output_nmsg.c | 287 ++++++++++++++++++++++++++++++++++++++++++++- nmsg/private.h | 8 ++ src/nmsgtool.c | 7 ++ src/nmsgtool.h | 2 +- 6 files changed, 364 insertions(+), 6 deletions(-) diff --git a/nmsg/output.c b/nmsg/output.c index 8813c6bc..d1b548cc 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -280,12 +280,21 @@ nmsg_output_write(nmsg_output_t output, nmsg_message_t msg) { nmsg_res nmsg_output_close(nmsg_output_t *output) { - nmsg_res res; + nmsg_res res, async_res; res = nmsg_res_success; switch ((*output)->type) { case nmsg_output_type_stream: res = _output_nmsg_flush(*output); + + /* + * Before random, fd and the locks below, all of which the + * compressor thread uses. + */ + async_res = _output_nmsg_async_destroy(*output); + if (res == nmsg_res_success) + res = async_res; + if ((*output)->stream->random != NULL) nmsg_random_destroy(&((*output)->stream->random)); #ifdef HAVE_LIBRDKAFKA @@ -409,6 +418,38 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { output->stream->do_zlib = zlibout; } +void +nmsg_output_set_zlib_async(nmsg_output_t output, bool async) { + nmsg_res res; + + /* + * The type test comes first because 'stream' is a union member: on a + * pres or json output, reading stream->type would reinterpret the + * bytes of a different struct rather than fail. + */ + if (output->type != nmsg_output_type_stream) + return; + if (output->stream->type != nmsg_stream_type_file) + return; + + /* + * Nothing to return an error through, so both paths are logged. Failing + * to start the compressor is survivable, but disabling it after writing + * can strand an error the worker had recorded. + */ + if (async) { + res = _output_nmsg_async_init(output); + if (res != nmsg_res_success) + _nmsg_dprintf(1, "%s: could not start compressor: %s\n", + __func__, nmsg_res_lookup(res)); + } else { + res = _output_nmsg_async_destroy(output); + if (res != nmsg_res_success) + _nmsg_dprintf(1, "%s: discarding pending write error: %s\n", + __func__, nmsg_res_lookup(res)); + } +} + void nmsg_output_set_endline(nmsg_output_t output, const char *endline) { if (output->type == nmsg_output_type_pres) { diff --git a/nmsg/output.h b/nmsg/output.h index f09ba6c1..98610518 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -382,4 +382,27 @@ nmsg_output_set_group(nmsg_output_t output, unsigned group); void nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); +/** + * Compress and write containers on a dedicated thread instead of on the thread + * calling nmsg_output_write(). Only affects file outputs. + * + * One container may be in flight at a time. A writer that finishes a container + * while the previous one is still being compressed blocks until it completes. + * + * Because a container is written after nmsg_output_write() returns, a write + * error is reported by a later nmsg_output_write(), or by nmsg_output_flush() + * or nmsg_output_close(), rather than by the call that supplied the data. + * + * Call before the first write. Enabling this later is harmless, but disabling + * it after writing discards any error already recorded for a container that + * has not yet been reported. + * + * \param[in] output nmsg_output_t object. + * + * \param[in] async True (compress on a separate thread) or false (compress + * inline, the default). + */ +void +nmsg_output_set_zlib_async(nmsg_output_t output, bool async); + #endif /* NMSG_OUTPUT_H */ diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index 13421e9b..6384c0b8 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -21,21 +21,121 @@ /* Forward. */ static nmsg_res container_write(nmsg_output_t, nmsg_container_t*); +static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool); static nmsg_res frag_write(nmsg_output_t, nmsg_container_t); static nmsg_res send_buffer(nmsg_output_t, uint8_t *buf, size_t len); +static nmsg_res async_drain(struct nmsg_ostr_async *); + +/* Data structures. */ + +/* + * One compressor thread and one handoff slot. The producer fills 'pending' and + * moves on; the worker takes it, compresses and writes it outside the lock. + * A producer finding the slot still occupied blocks, so a compressor that + * cannot keep up applies backpressure rather than growing a backlog. + */ +struct nmsg_ostr_async { + pthread_mutex_t lock; + pthread_cond_t work_ready; /* pending != NULL || shutdown */ + pthread_cond_t slot_free; /* pending == NULL && !busy */ + nmsg_container_t pending; + bool pending_frag; /* Overfull: needs frag_write(). */ + bool busy; /* Worker holds a container. */ + bool shutdown; + bool started; /* Worker exists; must be joined. */ + bool failed; /* pthread_create() failed; do not retry. */ + pthread_t worker; + nmsg_res first_error; /* Sticky; surfaced by flush. */ + nmsg_output_t output; + uint64_t n_blocked; /* Producer waits on a full slot. */ +}; /* Internal functions. */ +nmsg_res +_output_nmsg_async_init(nmsg_output_t output) { + struct nmsg_ostr_async *pool; + + if (output->stream->so_pool != NULL) + return (nmsg_res_success); + + pool = calloc(1, sizeof(*pool)); + if (pool == NULL) + return (nmsg_res_memfail); + + if (pthread_mutex_init(&pool->lock, NULL) != 0) + goto fail_mutex; + if (pthread_cond_init(&pool->work_ready, NULL) != 0) + goto fail_work_ready; + if (pthread_cond_init(&pool->slot_free, NULL) != 0) + goto fail_slot_free; + + pool->output = output; + output->stream->so_pool = pool; + + return (nmsg_res_success); + +fail_slot_free: + pthread_cond_destroy(&pool->work_ready); +fail_work_ready: + pthread_mutex_destroy(&pool->lock); +fail_mutex: + free(pool); + + return (nmsg_res_failure); +} + +/* + * Stop the compressor and reclaim it. Any pending container is written first. + * Must run before the stream's fd, random and locks go away, since the worker + * uses all of them. + */ +nmsg_res +_output_nmsg_async_destroy(nmsg_output_t output) { + struct nmsg_ostr_async *pool = output->stream->so_pool; + nmsg_res res; + bool started; + + if (pool == NULL) + return (nmsg_res_success); + + pthread_mutex_lock(&pool->lock); + pool->shutdown = true; /* Set under the lock: a worker about */ + started = pool->started; /* to wait would miss the wakeup. */ + pthread_cond_broadcast(&pool->work_ready); + pthread_cond_broadcast(&pool->slot_free); + pthread_mutex_unlock(&pool->lock); + + if (started) + pthread_join(pool->worker, NULL); + + if (pool->n_blocked > 0) + _nmsg_dprintf(2, "%s: producer waited on the compressor %" PRIu64 + " times\n", __func__, pool->n_blocked); + + /* Read after the join; the worker writes it up to the moment it exits. */ + res = pool->first_error; + + pthread_cond_destroy(&pool->slot_free); + pthread_cond_destroy(&pool->work_ready); + pthread_mutex_destroy(&pool->lock); + free(pool); + output->stream->so_pool = NULL; + + return (res); +} + nmsg_res _output_nmsg_flush(nmsg_output_t output) { nmsg_res res = nmsg_res_success; + nmsg_res drain_res; pthread_mutex_lock(&output->stream->c_lock); if (nmsg_container_get_num_payloads(output->stream->c) > 0) { /* Process container; container is destroyed. */ - res = container_write(output, &output->stream->c); + res = container_submit(output, &output->stream->c, false /* is_frag */); output->stream->c = nmsg_container_init(output->stream->bufsz); if (output->stream->c == NULL) @@ -47,6 +147,15 @@ _output_nmsg_flush(nmsg_output_t output) { pthread_mutex_unlock(&output->stream->c_lock); + /* + * A flush means the data has been written, so wait out anything the + * compressor is still holding. Done outside c_lock so producers are not + * held off for the length of a compression. + */ + drain_res = async_drain(output->stream->so_pool); + if (res == nmsg_res_success) + res = drain_res; + return (res); } @@ -121,16 +230,16 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ - res = container_write(output, &old_c); /* Write data from prior container. */ + res = container_submit(output, &old_c, false /* is_frag */); /* Write data from prior container. */ if (res != nmsg_res_success) return (res); /* Proceed to write current message to new container. */ goto retry; } else if (res == nmsg_res_success && is_buffered == false) { /* Includes current message. */ - res = container_write(output, &old_c); + res = container_submit(output, &old_c, false /* is_frag */); } else if (res == nmsg_res_container_overfull) { /* Includes current message. */ - res = frag_write(output, old_c); + res = container_submit(output, &old_c, true /* is_frag */); } return (res); @@ -138,6 +247,104 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { /* Private functions. */ +static void * +async_worker(void *arg) +{ + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + + pthread_mutex_lock(&pool->lock); + + /* + * Keep going while there is work, or until told to stop. Testing + * 'pending' first is what stops shutdown from discarding a container: + * a slot filled just before shutdown is still written. That happens on + * every clean SIGTERM, and dropping it would lose the tail of the + * final file. + */ + while (pool->pending != NULL || !pool->shutdown) { + nmsg_container_t co; + bool is_frag; + nmsg_res res; + + if (pool->pending == NULL) { + pthread_cond_wait(&pool->work_ready, &pool->lock); + continue; + } + + co = pool->pending; + is_frag = pool->pending_frag; + pool->pending = NULL; + pool->busy = true; + + pthread_cond_broadcast(&pool->slot_free); + pthread_mutex_unlock(&pool->lock); + + if (is_frag) + res = frag_write(pool->output, co); + else + res = container_write(pool->output, &co); + + pthread_mutex_lock(&pool->lock); + pool->busy = false; + if (res != nmsg_res_success && pool->first_error == nmsg_res_success) + pool->first_error = res; + + /* A drainer waits on !busy, not just on an empty slot. */ + pthread_cond_broadcast(&pool->slot_free); + } + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * Start the worker. Called under pool->lock on first submit rather than when + * the output is configured, because nmsgtool creates its outputs before it + * daemonizes, and daemonize() is a bare fork() which no thread survives. + * Starting on first write puts the worker in whichever process does the + * writing. + */ +static void +async_start(struct nmsg_ostr_async *pool) +{ + int pthread_res; + + pthread_res = pthread_create(&pool->worker, NULL, async_worker, pool); + if (pthread_res != 0) { + pool->failed = true; + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, + strerror(pthread_res)); + return; + } + pool->started = true; +} + +/* + * Wait until nothing is queued or in flight, and take any error the worker + * recorded. Under nmsg_io this cannot starve: check_close_event() holds + * io_output->refcount across the write and call_close_fp() waits for it to + * drop, so no other thread is inside nmsg_output_write() while a close runs. + * A caller driving nmsg_output_flush() directly from several threads has no + * such guarantee. + */ +static nmsg_res +async_drain(struct nmsg_ostr_async *pool) +{ + nmsg_res res; + + if (pool == NULL) + return (nmsg_res_success); + + pthread_mutex_lock(&pool->lock); + while (pool->pending != NULL || pool->busy) + pthread_cond_wait(&pool->slot_free, &pool->lock); + res = pool->first_error; + pool->first_error = nmsg_res_success; + pthread_mutex_unlock(&pool->lock); + + return (res); +} + /* * Send/write the contents of a container. * Container is destroyed, whether contents are successfully processed or not. @@ -166,6 +373,78 @@ container_write(nmsg_output_t output, nmsg_container_t *co) return (res); } +/* + * Hand a finished container to the compressor thread, or process it inline if + * there is no compressor. The container is consumed either way. + * + * The async path reports success for the container it just took, since that + * container has not been written yet. An error from an EARLIER container is + * returned here instead, which is what keeps a full disk fatal: write_file() + * returns nmsg_res_errno, io_thr_input() stops the loop on it (nmsg/io.c), + * and that is the only way nmsgtool notices ENOSPC. Delayed by one + * container rather than by a whole rotation. + */ +static nmsg_res +container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag) +{ + struct nmsg_ostr_async *pool = output->stream->so_pool; + nmsg_res res = nmsg_res_success; + bool handed_off = false; + + if (pool != NULL) { + bool blocked = false; + + pthread_mutex_lock(&pool->lock); + + if (!pool->started && !pool->failed && !pool->shutdown) + async_start(pool); + + if (pool->started) { + while (pool->pending != NULL && !pool->shutdown) { + if (!blocked) { + pool->n_blocked++; + blocked = true; + } + pthread_cond_wait(&pool->slot_free, &pool->lock); + } + + /* + * Re-tested after the wait: shutdown can arrive while + * blocked here. + */ + if (!pool->shutdown) { + pool->pending = *co; + pool->pending_frag = is_frag; + *co = NULL; + handed_off = true; + + res = pool->first_error; + pool->first_error = nmsg_res_success; + + pthread_cond_signal(&pool->work_ready); + } + } + + pthread_mutex_unlock(&pool->lock); + } + + if (handed_off) + return (res); + + if (is_frag) { + nmsg_container_t tmp = *co; + + /* + * frag_write() takes the container by value and destroys it + * internally, unlike container_write(). + */ + *co = NULL; + return (frag_write(output, tmp)); + } + + return (container_write(output, co)); +} + static nmsg_res write_sock(int fd, uint8_t *buf, size_t len) { diff --git a/nmsg/private.h b/nmsg/private.h index 42caf21e..57c36b1c 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -307,6 +307,11 @@ struct nmsg_stream_input { nmsg_input_stream_read_fp stream_read_fp; }; +/* + * Asynchronous compressor for a stream output. + */ +struct nmsg_ostr_async; + /* nmsg_stream_output: used by nmsg_output */ struct nmsg_stream_output { pthread_mutex_t c_lock; /* Container lock. */ @@ -331,6 +336,7 @@ struct nmsg_stream_output { bool do_sequence; atomic_uint_fast32_t so_sequence_num; uint64_t sequence_id; + struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ }; /* nmsg_callback_output: used by nmsg_output */ @@ -586,6 +592,8 @@ nmsg_output_t _output_open_kafka(void *s, size_t bufsz); /* from output_nmsg.c */ nmsg_res _output_nmsg_flush(nmsg_output_t); nmsg_res _output_nmsg_write(nmsg_output_t, nmsg_message_t); +nmsg_res _output_nmsg_async_init(nmsg_output_t); +nmsg_res _output_nmsg_async_destroy(nmsg_output_t); #ifdef HAVE_LIBRDKAFKA nmsg_res _output_kafka_payload_write(nmsg_output_t, nmsg_message_t); nmsg_res _output_kafka_payload_flush(nmsg_output_t); diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 3d870345..66ddb6de 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -326,6 +326,12 @@ static argv_t args[] = { NULL, "compress nmsg output" }, + { '\0', "zasync", + ARGV_BOOL, + &ctx.zasync, + NULL, + "compress file output on a separate thread" }, + { ARGV_LAST, 0, 0, 0, 0, 0 } }; @@ -436,6 +442,7 @@ setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_buffered(output, !(c->unbuffered)); nmsg_output_set_endline(output, c->endline_str); nmsg_output_set_zlibout(output, c->zlibout); + nmsg_output_set_zlib_async(output, c->zasync); nmsg_output_set_source(output, c->set_source); nmsg_output_set_operator(output, c->set_operator); nmsg_output_set_group(output, c->set_group); diff --git a/src/nmsgtool.h b/src/nmsgtool.h index 2d1c99b8..5a2d0430 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -56,7 +56,7 @@ typedef struct { argv_array_t r_nmsg, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; argv_array_t r_pcapfile, r_pcapif; argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; - bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; + bool help, mirror, unbuffered, zlibout, zasync, daemon, version, interval_randomized; char *endline, *kicker, *mname, *vname, *bpfstr, *filter_policy, *kafka_key_field; int debug, signal; unsigned mtu, count, interval, rate, freq, byte_rate; From 711cfdbeb428fd45e368af38ebe0b0fd6a90614a Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Thu, 20 Aug 2026 18:25:27 -0400 Subject: [PATCH 13/18] Widen --zasync to a compressor pool; add a committer that writes in ticket order --- debian/libnmsg8.symbols | 2 + doc/docbook/nmsgtool.docbook | 40 +++ nmsg/output.c | 26 +- nmsg/output.h | 24 ++ nmsg/output_nmsg.c | 561 ++++++++++++++++++++++++++--------- nmsg/private.h | 3 +- src/nmsgtool.c | 83 +++++- src/nmsgtool.h | 15 +- src/process_args.c | 8 + 9 files changed, 607 insertions(+), 155 deletions(-) diff --git a/debian/libnmsg8.symbols b/debian/libnmsg8.symbols index 33dd3b5a..a4a7eb0c 100644 --- a/debian/libnmsg8.symbols +++ b/debian/libnmsg8.symbols @@ -148,6 +148,8 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_set_operator@Base 0.5.0 nmsg_output_set_rate@Base 0.5.0 nmsg_output_set_source@Base 0.5.0 + nmsg_output_set_zlib_async@Base 0.5.0 + nmsg_output_set_zlib_workers@Base 0.5.0 nmsg_output_set_zlibout@Base 0.5.0 nmsg_output_write@Base 0.11.1 nmsg_pcap_filter@Base 0.6.5 diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 021afef6..d1f8be1e 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -656,6 +656,46 @@ + + n + + Compress written NMSG containers on + n separate threads rather than on + the thread that filled them. A thread that is compressing is + not reading its socket, and on a high volume channel that + pause is long enough for the kernel receive queue to + overflow; handing the container to a compressor lets the + reader carry straight on. + + A value of chooses a thread count: + two per input socket, since a reader thread can saturate + roughly one core compressing, bounded by the cores the + readers leave spare and never fewer than four, because a + single socket can carry several cores' worth on its own. + Sizing it exactly is not important, because + a reader that finds every compressor busy compresses the + container itself, exactly as it would with this option + unset. Too small a pool therefore degrades to the behaviour + of no pool at all rather than stalling, and neither setting + is slower than leaving the option off. + + This is worth using when a single input socket carries + more data than one core can compress. A channel spread over + a range of ports already compresses on every reader thread + and gains little; it also costs a little output size there, + because readers that never pause interleave their sources + more finely within a container, which compresses slightly + worse. With a single input socket the output is byte for + byte identical to compressing inline, whatever + n is. + + Containers are always written in the order they were + filled. Applies to file () outputs only, + and has no effect with + . + + + diff --git a/nmsg/output.c b/nmsg/output.c index d1b548cc..07cf54e9 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -420,6 +420,11 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { void nmsg_output_set_zlib_async(nmsg_output_t output, bool async) { + nmsg_output_set_zlib_workers(output, async ? 1 : 0); +} + +void +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { nmsg_res res; /* @@ -432,20 +437,31 @@ nmsg_output_set_zlib_async(nmsg_output_t output, bool async) { if (output->stream->type != nmsg_stream_type_file) return; + /* + * Unbuffered output flushes a container per message, so a pool would + * spend a ticket, a condvar signal and a wakeup per message to compress + * a single payload. + */ + if (workers > 0 && !output->stream->buffered) { + _nmsg_dprintf(1, "%s: ignored: not available on unbuffered output\n", + __func__); + return; + } + /* * Nothing to return an error through, so both paths are logged. Failing - * to start the compressor is survivable, but disabling it after writing - * can strand an error the worker had recorded. + * to start the pool is survivable, but disabling it after writing can + * strand an error a worker had recorded. */ - if (async) { - res = _output_nmsg_async_init(output); + if (workers > 0) { + res = _output_nmsg_async_init(output, workers); if (res != nmsg_res_success) _nmsg_dprintf(1, "%s: could not start compressor: %s\n", __func__, nmsg_res_lookup(res)); } else { res = _output_nmsg_async_destroy(output); if (res != nmsg_res_success) - _nmsg_dprintf(1, "%s: discarding pending write error: %s\n", + _nmsg_dprintf(1, "%s: compressor reported: %s\n", __func__, nmsg_res_lookup(res)); } } diff --git a/nmsg/output.h b/nmsg/output.h index 98610518..16e80829 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -405,4 +405,28 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); void nmsg_output_set_zlib_async(nmsg_output_t output, bool async); +/** + * Compress containers on a pool of worker threads instead of on the thread + * that filled them. + * + * A reader that compresses is not reading, and on a busy channel that pause is + * long enough for the socket to overflow. With a pool the reader hands the + * container over and returns to reading; if every worker is busy it compresses + * the container itself, exactly as it would with no pool, so this is never + * slower than leaving it off. + * + * Writes stay in the order the containers were filled, so the output is + * byte-identical whatever \a workers is set to. + * + * File outputs only, and only when buffered. Call before the first write. + * + * \param[in] output nmsg_output_t object. + * + * \param[in] workers Number of compressor threads, or 0 to compress inline + * (the default). More than one is only useful when a single output is + * offered more data than one core can compress. + */ +void +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); + #endif /* NMSG_OUTPUT_H */ diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index 6384c0b8..a9e8a370 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -21,7 +21,7 @@ /* Forward. */ static nmsg_res container_write(nmsg_output_t, nmsg_container_t*); -static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool); +static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool, uint64_t); static nmsg_res frag_write(nmsg_output_t, nmsg_container_t); static nmsg_res send_buffer(nmsg_output_t, uint8_t *buf, size_t len); static nmsg_res async_drain(struct nmsg_ostr_async *); @@ -29,96 +29,176 @@ static nmsg_res async_drain(struct nmsg_ostr_async *); /* Data structures. */ /* - * One compressor thread and one handoff slot. The producer fills 'pending' and - * moves on; the worker takes it, compresses and writes it outside the lock. - * A producer finding the slot still occupied blocks, so a compressor that - * cannot keep up applies backpressure rather than growing a backlog. + * A compressor pool: a ring of slots, nworkers compressor threads and one + * committer thread. + * + * A container is given a ticket when it is sealed, under c_lock, so tickets + * follow the order the containers were closed in. Slot i serves every ticket + * with (ticket % depth) == i, so the producer of ticket T waits only for + * ticket T - depth to have been written. + * + * Compression runs on whichever thread is free. Only the committer writes, and + * only in ticket order, so the byte stream is identical to the synchronous + * path however many workers are running. + * + * A producer that finds every worker busy compresses the container itself + * rather than waiting. That is exactly what the synchronous path does, so the + * pool is never slower than no pool at all. */ +typedef enum { + slot_empty = 0, /* Free. */ + slot_work, /* Container waiting for a compressor. */ + slot_taken, /* A worker or a producer is compressing it. */ + slot_done, /* Compressed, waiting its turn to be written. */ + slot_frag /* Oversized: the committer must fragment it itself. */ +} async_slot_state; + +struct async_slot { + async_slot_state state; + nmsg_container_t co; /* slot_work, slot_frag */ + uint8_t *buf; /* slot_done */ + size_t buf_len; + nmsg_res res; +}; + struct nmsg_ostr_async { pthread_mutex_t lock; - pthread_cond_t work_ready; /* pending != NULL || shutdown */ - pthread_cond_t slot_free; /* pending == NULL && !busy */ - nmsg_container_t pending; - bool pending_frag; /* Overfull: needs frag_write(). */ - bool busy; /* Worker holds a container. */ + pthread_cond_t work_ready; /* A slot became slot_work. */ + pthread_cond_t commit_ready; /* The committer's slot is ready. */ + pthread_cond_t slot_free; /* A slot became slot_empty. */ + struct async_slot *slots; + unsigned depth; + unsigned nworkers; + unsigned busy; /* Workers currently compressing. */ + uint64_t issued; /* Highest ticket claimed, plus one. */ + uint64_t commit_next; /* Ticket allowed to write now. */ bool shutdown; - bool started; /* Worker exists; must be joined. */ - bool failed; /* pthread_create() failed; do not retry. */ - pthread_t worker; + bool started; + bool failed; /* Thread creation failed; stay inline. */ + pthread_t *workers; + pthread_t committer; nmsg_res first_error; /* Sticky; surfaced by flush. */ nmsg_output_t output; - uint64_t n_blocked; /* Producer waits on a full slot. */ + uint64_t n_inline; /* Containers a producer compressed. */ + uint64_t n_waited; /* Producers that waited for a slot. */ }; /* Internal functions. */ nmsg_res -_output_nmsg_async_init(nmsg_output_t output) { +_output_nmsg_async_init(nmsg_output_t output, unsigned nworkers) { struct nmsg_ostr_async *pool; + unsigned depth; + + if (nworkers == 0) + return (nmsg_res_success); if (output->stream->so_pool != NULL) return (nmsg_res_success); + /* + * One slot per worker to compress in, plus a fixed margin for + * producers to deposit into and for finished buffers waiting their + * turn to be written. The margin does not need to scale with the + * worker count: a producer that cannot get a slot compresses the + * container itself rather than waiting, so a tight ring costs a little + * of the parallelism and never blocks a reader. + * + * A slot holds a container's payloads in memory, which is not the + * 1 MiB serialized size -- a channel of 30-byte payloads puts ~37k of + * them in a container, over 5 MB. That is what bounds the ring, and + * why the automatic worker count is capped. + */ + depth = nworkers + 8; + pool = calloc(1, sizeof(*pool)); if (pool == NULL) return (nmsg_res_memfail); + pool->slots = calloc(depth, sizeof(*pool->slots)); + if (pool->slots == NULL) + goto fail_slots; + + pool->workers = calloc(nworkers, sizeof(*pool->workers)); + if (pool->workers == NULL) + goto fail_workers; + if (pthread_mutex_init(&pool->lock, NULL) != 0) goto fail_mutex; if (pthread_cond_init(&pool->work_ready, NULL) != 0) goto fail_work_ready; + if (pthread_cond_init(&pool->commit_ready, NULL) != 0) + goto fail_commit_ready; if (pthread_cond_init(&pool->slot_free, NULL) != 0) goto fail_slot_free; + pool->depth = depth; + pool->nworkers = nworkers; pool->output = output; output->stream->so_pool = pool; return (nmsg_res_success); fail_slot_free: + pthread_cond_destroy(&pool->commit_ready); +fail_commit_ready: pthread_cond_destroy(&pool->work_ready); fail_work_ready: pthread_mutex_destroy(&pool->lock); fail_mutex: + free(pool->workers); +fail_workers: + free(pool->slots); +fail_slots: free(pool); return (nmsg_res_failure); } /* - * Stop the compressor and reclaim it. Any pending container is written first. - * Must run before the stream's fd, random and locks go away, since the worker - * uses all of them. + * Stop the pool and reclaim it. Everything still queued is written first. + * Must run before the stream's fd, random and locks go away, since the threads + * use all of them. */ nmsg_res _output_nmsg_async_destroy(nmsg_output_t output) { struct nmsg_ostr_async *pool = output->stream->so_pool; nmsg_res res; bool started; + unsigned i; if (pool == NULL) return (nmsg_res_success); pthread_mutex_lock(&pool->lock); - pool->shutdown = true; /* Set under the lock: a worker about */ + pool->shutdown = true; /* Set under the lock: a thread about */ started = pool->started; /* to wait would miss the wakeup. */ pthread_cond_broadcast(&pool->work_ready); + pthread_cond_broadcast(&pool->commit_ready); pthread_cond_broadcast(&pool->slot_free); pthread_mutex_unlock(&pool->lock); - if (started) - pthread_join(pool->worker, NULL); + if (started) { + for (i = 0; i < pool->nworkers; i++) + pthread_join(pool->workers[i], NULL); + pthread_join(pool->committer, NULL); + } - if (pool->n_blocked > 0) - _nmsg_dprintf(2, "%s: producer waited on the compressor %" PRIu64 - " times\n", __func__, pool->n_blocked); + if (pool->n_inline > 0 || pool->n_waited > 0) + _nmsg_dprintf(2, "%s: %u worker(s); %" PRIu64 " container(s) " + "compressed by the reader, %" PRIu64 " wait(s) for " + "a free slot\n", __func__, pool->nworkers, + pool->n_inline, pool->n_waited); - /* Read after the join; the worker writes it up to the moment it exits. */ + /* Read after the joins; the threads write it until they exit. */ res = pool->first_error; pthread_cond_destroy(&pool->slot_free); + pthread_cond_destroy(&pool->commit_ready); pthread_cond_destroy(&pool->work_ready); pthread_mutex_destroy(&pool->lock); + free(pool->workers); + free(pool->slots); free(pool); output->stream->so_pool = NULL; @@ -127,32 +207,45 @@ _output_nmsg_async_destroy(nmsg_output_t output) { nmsg_res _output_nmsg_flush(nmsg_output_t output) { + struct nmsg_stream_output *ostr = output->stream; nmsg_res res = nmsg_res_success; nmsg_res drain_res; + nmsg_container_t old_c = NULL; + uint64_t ticket = 0; - pthread_mutex_lock(&output->stream->c_lock); + pthread_mutex_lock(&ostr->c_lock); - if (nmsg_container_get_num_payloads(output->stream->c) > 0) { + if (nmsg_container_get_num_payloads(ostr->c) > 0) { + old_c = ostr->c; + ticket = ostr->so_ticket++; - /* Process container; container is destroyed. */ - res = container_submit(output, &output->stream->c, false /* is_frag */); - - output->stream->c = nmsg_container_init(output->stream->bufsz); - if (output->stream->c == NULL) + ostr->c = nmsg_container_init(ostr->bufsz); + if (ostr->c == NULL) res = nmsg_res_memfail; else - nmsg_container_set_sequence(output->stream->c, output->stream->do_sequence); - + nmsg_container_set_sequence(ostr->c, ostr->do_sequence); } - pthread_mutex_unlock(&output->stream->c_lock); + pthread_mutex_unlock(&ostr->c_lock); + + /* + * Submitted outside c_lock. Flush runs on every file rotation, and a + * submit can wait for a free slot; doing that under c_lock would hold + * off every reader thread for the length of a compression. + */ + if (old_c != NULL) { + nmsg_res sub_res; + + sub_res = container_submit(output, &old_c, false, ticket); + if (res == nmsg_res_success) + res = sub_res; + } /* * A flush means the data has been written, so wait out anything the - * compressor is still holding. Done outside c_lock so producers are not - * held off for the length of a compression. + * pool is still holding. */ - drain_res = async_drain(output->stream->so_pool); + drain_res = async_drain(ostr->so_pool); if (res == nmsg_res_success) res = drain_res; @@ -165,6 +258,7 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { struct nmsg_stream_output *ostr = output->stream; nmsg_container_t old_c, new_c; nmsg_res res; + uint64_t ticket = 0; bool must_flush, is_buffered; assert(msg->np != NULL); @@ -221,6 +315,15 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { old_c = ostr->c; /* Process old, proceed with new. */ ostr->c = new_c; + + /* + * Ticket taken here, under c_lock, because this is where the + * container's contents become final. Taking it in + * container_submit() would order the containers by which + * thread won the race after the unlock, which is not the + * order they were filled in. + */ + ticket = ostr->so_ticket++; } pthread_mutex_unlock(&ostr->c_lock); /* Release locked container to other threads. */ @@ -230,16 +333,16 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ - res = container_submit(output, &old_c, false /* is_frag */); /* Write data from prior container. */ + res = container_submit(output, &old_c, false, ticket); /* Write data from prior container. */ if (res != nmsg_res_success) return (res); /* Proceed to write current message to new container. */ goto retry; } else if (res == nmsg_res_success && is_buffered == false) { /* Includes current message. */ - res = container_submit(output, &old_c, false /* is_frag */); + res = container_submit(output, &old_c, false, ticket); } else if (res == nmsg_res_container_overfull) { /* Includes current message. */ - res = container_submit(output, &old_c, true /* is_frag */); + res = container_submit(output, &old_c, true, ticket); } return (res); @@ -247,6 +350,75 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { /* Private functions. */ +/* + * Compress one container into a standalone buffer. The container is destroyed + * either way, as container_write() does. + */ +static nmsg_res +container_compress(nmsg_output_t output, nmsg_container_t *co, + uint8_t **buf, size_t *buf_len) +{ + struct nmsg_stream_output *ostr = output->stream; + nmsg_res res; + uint32_t seq; + uint8_t *shrunk; + + /* Multiple threads can enter here at once. */ + seq = atomic_fetch_add_explicit(&ostr->so_sequence_num, 1, memory_order_relaxed); + + res = nmsg_container_serialize(*co, buf, buf_len, true, /* do_header */ + ostr->do_zlib, seq, ostr->sequence_id); + nmsg_container_destroy(co); + + if (res != nmsg_res_success) { + *buf = NULL; + *buf_len = 0; + return (res); + } + + /* + * nmsg_container_serialize() returns the base of an allocation sized + * for the worst case, twice the unpacked estimate. A slot holds that + * until every earlier ticket has been written, so hand the rest back. + */ + shrunk = realloc(*buf, *buf_len); + if (shrunk != NULL) + *buf = shrunk; + + return (res); +} + +/* + * Send/write the contents of a container. + * Container is destroyed, whether contents are successfully processed or not. + */ +static nmsg_res +container_write(nmsg_output_t output, nmsg_container_t *co) +{ + nmsg_res res; + size_t buf_len; + uint8_t *buf; + + res = container_compress(output, co, &buf, &buf_len); + if (res != nmsg_res_success) + return (res); + + return (send_buffer(output, buf, buf_len)); +} + +/* Note an error, keeping the first one seen. Caller holds pool->lock. */ +static void +async_record_error(struct nmsg_ostr_async *pool, nmsg_res res) +{ + if (res != nmsg_res_success && pool->first_error == nmsg_res_success) + pool->first_error = res; +} + +/* + * Compressor thread. Takes any slot that needs compressing, in whatever order + * they become ready: compression order does not matter, only write order does, + * and the committer enforces that. + */ static void * async_worker(void *arg) { @@ -254,76 +426,169 @@ async_worker(void *arg) pthread_mutex_lock(&pool->lock); - /* - * Keep going while there is work, or until told to stop. Testing - * 'pending' first is what stops shutdown from discarding a container: - * a slot filled just before shutdown is still written. That happens on - * every clean SIGTERM, and dropping it would lose the tail of the - * final file. - */ - while (pool->pending != NULL || !pool->shutdown) { + for (;;) { + struct async_slot *slot = NULL; nmsg_container_t co; - bool is_frag; + uint8_t *buf; + size_t buf_len; nmsg_res res; + unsigned i; - if (pool->pending == NULL) { + for (i = 0; i < pool->depth; i++) { + if (pool->slots[i].state == slot_work) { + slot = &pool->slots[i]; + break; + } + } + + if (slot == NULL) { + /* + * Nothing to compress. Exit only once the producers + * have stopped, so a container queued just before + * shutdown is still compressed and written; that + * happens on every clean SIGTERM. + */ + if (pool->shutdown) + break; pthread_cond_wait(&pool->work_ready, &pool->lock); continue; } - co = pool->pending; - is_frag = pool->pending_frag; - pool->pending = NULL; - pool->busy = true; - - pthread_cond_broadcast(&pool->slot_free); + slot->state = slot_taken; + co = slot->co; + slot->co = NULL; + pool->busy++; pthread_mutex_unlock(&pool->lock); - if (is_frag) - res = frag_write(pool->output, co); - else - res = container_write(pool->output, &co); + res = container_compress(pool->output, &co, &buf, &buf_len); pthread_mutex_lock(&pool->lock); - pool->busy = false; - if (res != nmsg_res_success && pool->first_error == nmsg_res_success) - pool->first_error = res; + pool->busy--; + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); + } + + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * The only thread that writes. It takes tickets strictly in order, so the file + * is byte-identical to what the synchronous path would have produced no matter + * how many workers compressed in parallel. + * + * It is also the only caller of frag_write(). + */ +static void * +async_committer(void *arg) +{ + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + + pthread_mutex_lock(&pool->lock); - /* A drainer waits on !busy, not just on an empty slot. */ + for (;;) { + struct async_slot *slot = &pool->slots[pool->commit_next % pool->depth]; + async_slot_state state = slot->state; + nmsg_res res; + + if (state == slot_done) { + uint8_t *buf = slot->buf; + size_t buf_len = slot->buf_len; + + res = slot->res; + slot->buf = NULL; + pthread_mutex_unlock(&pool->lock); + + /* send_buffer() owns buf from here, error or not. */ + if (res == nmsg_res_success) + res = send_buffer(pool->output, buf, buf_len); + + pthread_mutex_lock(&pool->lock); + } else if (state == slot_frag) { + nmsg_container_t co = slot->co; + + slot->co = NULL; + pthread_mutex_unlock(&pool->lock); + + /* frag_write() takes the container by value and destroys it. */ + res = frag_write(pool->output, co); + + pthread_mutex_lock(&pool->lock); + } else { + /* + * The next ticket is not ready. Exit only when the + * producers have stopped and every ticket they issued + * has been written. + */ + if (pool->shutdown && pool->commit_next >= pool->issued) + break; + pthread_cond_wait(&pool->commit_ready, &pool->lock); + continue; + } + + async_record_error(pool, res); + slot->state = slot_empty; + pool->commit_next++; pthread_cond_broadcast(&pool->slot_free); } + pthread_mutex_unlock(&pool->lock); return (NULL); } /* - * Start the worker. Called under pool->lock on first submit rather than when + * Start the threads. Called under pool->lock on first submit rather than when * the output is configured, because nmsgtool creates its outputs before it * daemonizes, and daemonize() is a bare fork() which no thread survives. - * Starting on first write puts the worker in whichever process does the + * Starting on first write puts the threads in whichever process does the * writing. */ static void async_start(struct nmsg_ostr_async *pool) { + unsigned i; int pthread_res; - pthread_res = pthread_create(&pool->worker, NULL, async_worker, pool); - if (pthread_res != 0) { - pool->failed = true; - _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, - strerror(pthread_res)); - return; + for (i = 0; i < pool->nworkers; i++) { + pthread_res = pthread_create(&pool->workers[i], NULL, async_worker, pool); + if (pthread_res != 0) + goto fail; } + + pthread_res = pthread_create(&pool->committer, NULL, async_committer, pool); + if (pthread_res != 0) + goto fail; + pool->started = true; + return; + +fail: + /* + * Stop whatever did start and fall back to compressing inline. Joining + * needs the lock released, and the caller still holds a container, so + * this stays simple: the threads created so far see shutdown and exit, + * and destroy joins them. + */ + pool->nworkers = i; + pool->failed = true; + pool->shutdown = true; + pool->started = (i > 0); + pthread_cond_broadcast(&pool->work_ready); + pthread_cond_broadcast(&pool->commit_ready); + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, + strerror(pthread_res)); } /* - * Wait until nothing is queued or in flight, and take any error the worker - * recorded. Under nmsg_io this cannot starve: check_close_event() holds - * io_output->refcount across the write and call_close_fp() waits for it to - * drop, so no other thread is inside nmsg_output_write() while a close runs. + * Wait until every ticket issued so far has been written, and take any error + * the pool recorded. Under nmsg_io this cannot starve: check_close_event() + * holds io_output->refcount across the write and call_close_fp() waits for it + * to drop, so no other thread is inside nmsg_output_write() while a close runs. * A caller driving nmsg_output_flush() directly from several threads has no * such guarantee. */ @@ -336,7 +601,7 @@ async_drain(struct nmsg_ostr_async *pool) return (nmsg_res_success); pthread_mutex_lock(&pool->lock); - while (pool->pending != NULL || pool->busy) + while (pool->started && !pool->failed && pool->commit_next < pool->issued) pthread_cond_wait(&pool->slot_free, &pool->lock); res = pool->first_error; pool->first_error = nmsg_res_success; @@ -346,91 +611,99 @@ async_drain(struct nmsg_ostr_async *pool) } /* - * Send/write the contents of a container. - * Container is destroyed, whether contents are successfully processed or not. - */ -static nmsg_res -container_write(nmsg_output_t output, nmsg_container_t *co) -{ - nmsg_res res; - size_t buf_len; - uint32_t seq; - uint8_t *buf; - - /* Multiple threads can enter here at once. */ - seq = atomic_fetch_add_explicit(&output->stream->so_sequence_num, 1, memory_order_relaxed); - - res = nmsg_container_serialize(*co, &buf, &buf_len, true, /* do_header */ - output->stream->do_zlib, seq, output->stream->sequence_id); - - if (res != nmsg_res_success) - goto out; - - res = send_buffer(output, buf, buf_len); -out: - nmsg_container_destroy(co); - - return (res); -} - -/* - * Hand a finished container to the compressor thread, or process it inline if - * there is no compressor. The container is consumed either way. + * Hand a finished container to the pool, or process it inline if there is no + * pool. The container is consumed either way. * - * The async path reports success for the container it just took, since that + * The pool reports success for the container it just took, since that * container has not been written yet. An error from an EARLIER container is - * returned here instead, which is what keeps a full disk fatal: write_file() - * returns nmsg_res_errno, io_thr_input() stops the loop on it (nmsg/io.c), - * and that is the only way nmsgtool notices ENOSPC. Delayed by one - * container rather than by a whole rotation. + * returned here. */ static nmsg_res -container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag) +container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag, + uint64_t ticket) { struct nmsg_ostr_async *pool = output->stream->so_pool; + struct async_slot *slot; nmsg_res res = nmsg_res_success; - bool handed_off = false; - - if (pool != NULL) { - bool blocked = false; + uint8_t *buf; + size_t buf_len; + bool inline_compress = false; - pthread_mutex_lock(&pool->lock); + if (pool == NULL) + goto inline_path; - if (!pool->started && !pool->failed && !pool->shutdown) - async_start(pool); + pthread_mutex_lock(&pool->lock); - if (pool->started) { - while (pool->pending != NULL && !pool->shutdown) { - if (!blocked) { - pool->n_blocked++; - blocked = true; - } - pthread_cond_wait(&pool->slot_free, &pool->lock); - } + if (!pool->started && !pool->failed && !pool->shutdown) + async_start(pool); - /* - * Re-tested after the wait: shutdown can arrive while - * blocked here. - */ - if (!pool->shutdown) { - pool->pending = *co; - pool->pending_frag = is_frag; - *co = NULL; - handed_off = true; + if (!pool->started || pool->failed || pool->shutdown) { + pthread_mutex_unlock(&pool->lock); + goto inline_path; + } - res = pool->first_error; - pool->first_error = nmsg_res_success; + slot = &pool->slots[ticket % pool->depth]; - pthread_cond_signal(&pool->work_ready); - } + /* + * Wait for the slot this ticket owns, which the ticket 'depth' earlier + * releases when it is written. Only reached when the writer has fallen + * a whole ring behind. + */ + if (slot->state != slot_empty) { + pool->n_waited++; + while (slot->state != slot_empty && !pool->shutdown) + pthread_cond_wait(&pool->slot_free, &pool->lock); + + if (pool->shutdown) { + pthread_mutex_unlock(&pool->lock); + goto inline_path; } + } + + if (ticket >= pool->issued) + pool->issued = ticket + 1; + + if (is_frag) { + /* Only the committer fragments; see async_committer(). */ + slot->co = *co; + *co = NULL; + slot->state = slot_frag; + pthread_cond_broadcast(&pool->commit_ready); + } else if (pool->busy < pool->nworkers) { + /* A worker is free: hand it over and get back to reading. */ + slot->co = *co; + *co = NULL; + slot->state = slot_work; + pthread_cond_signal(&pool->work_ready); + } else { + /* + * Every worker is busy. Compress it here rather than wait then deposit + * the result and return. + */ + slot->state = slot_taken; + pool->n_inline++; + inline_compress = true; + } + res = pool->first_error; + pool->first_error = nmsg_res_success; + pthread_mutex_unlock(&pool->lock); + + if (inline_compress) { + nmsg_res c_res = container_compress(output, co, &buf, &buf_len); + + pthread_mutex_lock(&pool->lock); + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = c_res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); pthread_mutex_unlock(&pool->lock); } - if (handed_off) - return (res); + return (res); +inline_path: if (is_frag) { nmsg_container_t tmp = *co; diff --git a/nmsg/private.h b/nmsg/private.h index 57c36b1c..56854576 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -336,6 +336,7 @@ struct nmsg_stream_output { bool do_sequence; atomic_uint_fast32_t so_sequence_num; uint64_t sequence_id; + uint64_t so_ticket; /* Next container ticket; c_lock. */ struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ }; @@ -592,7 +593,7 @@ nmsg_output_t _output_open_kafka(void *s, size_t bufsz); /* from output_nmsg.c */ nmsg_res _output_nmsg_flush(nmsg_output_t); nmsg_res _output_nmsg_write(nmsg_output_t, nmsg_message_t); -nmsg_res _output_nmsg_async_init(nmsg_output_t); +nmsg_res _output_nmsg_async_init(nmsg_output_t, unsigned); nmsg_res _output_nmsg_async_destroy(nmsg_output_t); #ifdef HAVE_LIBRDKAFKA nmsg_res _output_kafka_payload_write(nmsg_output_t, nmsg_message_t); diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 66ddb6de..63587b9f 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -327,10 +327,10 @@ static argv_t args[] = { "compress nmsg output" }, { '\0', "zasync", - ARGV_BOOL, + ARGV_INT, &ctx.zasync, - NULL, - "compress file output on a separate thread" }, + "n", + "compress file output on n threads (-1 chooses)" }, { ARGV_LAST, 0, 0, 0, 0, 0 } }; @@ -365,6 +365,7 @@ int main(int argc, char **argv) { #endif /* HAVE_LIBZMQ */ ctx.statsmods_loaded = statsmod_vec_init(1); + ctx.initial_outputs = output_vec_init(1); /* initialize the nmsg_io engine */ ctx.io = nmsg_io_init(); @@ -437,15 +438,89 @@ usage(const char *msg) { exit(msg == NULL ? EXIT_SUCCESS : EXIT_FAILURE); } +/* + * How many compressor threads an output should get. + * + * A negative --zasync means auto. Two things bound the answer: + * + * Demand scales with the input sockets. Each reader thread can saturate + * roughly one core compressing, and covering that takes about two workers + * per socket. + * + * Supply is the cores the readers leave. Workers beyond that only contend. + * + * The floor matters because one socket can carry several cores' worth on its + * own, so a count that merely followed the socket count would under-serve + * exactly the case the pool exists for. + * + * Getting it wrong is cheap in one direction: a reader that finds every worker + * busy compresses the container itself, so too small a pool degrades to the + * behaviour of no pool rather than stalling. Too large only wastes idle + * threads and ring slots, and a slot holds a container's payloads in memory. + */ +static unsigned +zworkers_count(nmsgtool_ctx *c) { + long ncpu; + int n, spare; + + if (c->zasync == 0) + return (0); + if (c->zasync > 0) + return ((unsigned) c->zasync); + + ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if (ncpu < 1) + ncpu = 1; + + n = 2 * c->n_inputs; + + spare = (int) ncpu - c->n_inputs; + if (n > spare) + n = spare; + if (n < NMSGTOOL_ZWORKERS_MIN) + n = NMSGTOOL_ZWORKERS_MIN; + + return ((unsigned) n); +} + +/* + * Resolve -W and apply it to the outputs that already exist. Deferred to the + * end of process_args() because the input count is not final until then: a + * channel alias (-C) expands to its sockets after the outputs have been + * created, which is exactly the case the pool is sized for. + */ +void +setup_nmsg_output_workers(nmsgtool_ctx *c) { + size_t i; + + c->zworkers_resolved = zworkers_count(c); + + if (c->initial_outputs != NULL) { + for (i = 0; i < output_vec_size(c->initial_outputs); i++) + nmsg_output_set_zlib_workers( + output_vec_data(c->initial_outputs)[i], + c->zworkers_resolved); + output_vec_destroy(&c->initial_outputs); + } + + if (c->zworkers_resolved > 0 && c->debug >= 2) + fprintf(stderr, "%s: compressing on %u thread(s)\n", + argv_program, c->zworkers_resolved); +} + void setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_buffered(output, !(c->unbuffered)); nmsg_output_set_endline(output, c->endline_str); nmsg_output_set_zlibout(output, c->zlibout); - nmsg_output_set_zlib_async(output, c->zasync); + nmsg_output_set_zlib_workers(output, c->zworkers_resolved); nmsg_output_set_source(output, c->set_source); nmsg_output_set_operator(output, c->set_operator); nmsg_output_set_group(output, c->set_group); + + /* Outputs made before -W is resolved; see setup_nmsg_output_workers(). */ + if (c->initial_outputs != NULL) + output_vec_add(c->initial_outputs, output); } void diff --git a/src/nmsgtool.h b/src/nmsgtool.h index 5a2d0430..58a7c88f 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -49,6 +49,15 @@ union nmsgtool_sockaddr { typedef union nmsgtool_sockaddr nmsgtool_sockaddr; VECTOR_GENERATE(statsmod_vec, nmsg_statsmod_t) +VECTOR_GENERATE(output_vec, nmsg_output_t) + +/* + * Floor for an automatically chosen compressor pool. A single input socket can + * carry well over one core's worth of compression on its own -- at 120 MB/s it + * needed four workers, where two still lost 13 % -- so the count cannot simply + * follow the socket count downwards. + */ +#define NMSGTOOL_ZWORKERS_MIN 4 typedef struct { /* parameters */ @@ -56,7 +65,8 @@ typedef struct { argv_array_t r_nmsg, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; argv_array_t r_pcapfile, r_pcapif; argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; - bool help, mirror, unbuffered, zlibout, zasync, daemon, version, interval_randomized; + bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; + int zasync; /* Compressor threads; -1 chooses. */ char *endline, *kicker, *mname, *vname, *bpfstr, *filter_policy, *kafka_key_field; int debug, signal; unsigned mtu, count, interval, rate, freq, byte_rate; @@ -68,6 +78,8 @@ typedef struct { /* state */ char *endline_str; int n_inputs, n_outputs; + unsigned zworkers_resolved; /* 0 until process_args() finishes. */ + output_vec *initial_outputs; /* Non-NULL only during process_args(). */ nmsg_io_t io; #ifdef HAVE_LIBZMQ void *zmq_ctx; @@ -140,6 +152,7 @@ void pidfile_write(FILE *); void process_args(nmsgtool_ctx *); void setup_nmsg_input(nmsgtool_ctx *, nmsg_input_t); void setup_nmsg_output(nmsgtool_ctx *, nmsg_output_t); +void setup_nmsg_output_workers(nmsgtool_ctx *); void usage(const char *); #endif /* NMSGTOOL_H */ diff --git a/src/process_args.c b/src/process_args.c index c9c49419..e8c32afe 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -409,6 +409,14 @@ process_args(nmsgtool_ctx *c) { add_pres_output(c, "-"); } + /* + * Size the compressor pool now that every input exists. Must follow + * the implicit output above, and must precede daemonize(): the pool's + * threads are started on first write, which happens in whichever + * process ends up doing the writing. + */ + setup_nmsg_output_workers(c); + /* daemonize if necessary */ if (c->daemon) { if (!daemonize()) { From 8847deb5ca3994ea97b6780cf69759beeda08bc1 Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Fri, 21 Aug 2026 11:35:46 -0400 Subject: [PATCH 14/18] Move the compressor pool to output_async.c; spawn workers on demand, add tests --- Makefile.am | 11 + doc/docbook/nmsgtool.docbook | 37 +- nmsg/output.c | 14 +- nmsg/output.h | 43 +-- nmsg/output_async.c | 631 +++++++++++++++++++++++++++++++++++ nmsg/output_nmsg.c | 544 ++++-------------------------- nmsg/private.h | 18 +- src/io.c | 1 + src/nmsgtool.c | 65 +++- src/nmsgtool.h | 9 + src/process_args.c | 4 + tests/test-zpool-mt.c | 232 +++++++++++++ tests/test-zpool-order.c | 289 ++++++++++++++++ 13 files changed, 1346 insertions(+), 552 deletions(-) create mode 100644 nmsg/output_async.c create mode 100644 tests/test-zpool-mt.c create mode 100644 tests/test-zpool-order.c diff --git a/Makefile.am b/Makefile.am index e12d51a0..f5aa64ee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -217,6 +217,7 @@ LIBNMSG_LIB_MODULES = \ nmsg/nmsg.c \ nmsg/output.c \ nmsg/output_json.c \ + nmsg/output_async.c \ nmsg/output_nmsg.c \ nmsg/output_pres.c \ nmsg/payload.c \ @@ -561,6 +562,16 @@ check_PROGRAMS += tests/test-nmsg_output_set_rate tests_test_nmsg_output_set_rate_SOURCES = tests/test-nmsg_output_set_rate.c tests_test_nmsg_output_set_rate_LDADD = nmsg/libnmsg.la +TESTS += tests/test-zpool-order +check_PROGRAMS += tests/test-zpool-order +tests_test_zpool_order_SOURCES = tests/test-zpool-order.c +tests_test_zpool_order_LDADD = nmsg/libnmsg.la + +TESTS += tests/test-zpool-mt +check_PROGRAMS += tests/test-zpool-mt +tests_test_zpool_mt_SOURCES = tests/test-zpool-mt.c +tests_test_zpool_mt_LDADD = nmsg/libnmsg.la + DISTCLEANFILES += tests/group-operator-source-tests/test*.out DISTCLEANFILES += tests/nmsg-dns-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index d1f8be1e..accdc4bc 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -667,26 +667,35 @@ overflow; handing the container to a compressor lets the reader carry straight on. - A value of chooses a thread count: - two per input socket, since a reader thread can saturate - roughly one core compressing, bounded by the cores the - readers leave spare and never fewer than four, because a - single socket can carry several cores' worth on its own. - Sizing it exactly is not important, because - a reader that finds every compressor busy compresses the - container itself, exactly as it would with this option + n is a ceiling rather than + a thread count: compressors are started only as load calls + for them, so an output that never saturates never pays for + them. Values above twice the number of cores available to + the process are rejected, and a further internal limit + applies. + + A value of chooses the ceiling: + two per input, since a reader thread can saturate roughly + one core compressing, bounded by the cores the readers leave + spare, never fewer than four because a single input can + carry several cores' worth on its own, and then divided + across the file outputs, which each get their own pool and + share the same cores. Sizing it exactly is not important, + because a reader that finds every compressor busy compresses + the container itself, exactly as it would with this option unset. Too small a pool therefore degrades to the behaviour of no pool at all rather than stalling, and neither setting is slower than leaving the option off. - This is worth using when a single input socket carries - more data than one core can compress. A channel spread over - a range of ports already compresses on every reader thread - and gains little; it also costs a little output size there, + This is worth using when a single input carries more + data than one core can compress, and on file-to-file work + such as recompressing a capture. A channel spread over a + range of ports already compresses on every reader thread and + gains little; it also costs a little output size there, because readers that never pause interleave their sources more finely within a container, which compresses slightly - worse. With a single input socket the output is byte for - byte identical to compressing inline, whatever + worse. With a single input the output is byte for byte + identical to compressing inline, whatever n is. Containers are always written in the order they were diff --git a/nmsg/output.c b/nmsg/output.c index 07cf54e9..36003960 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -291,7 +291,7 @@ nmsg_output_close(nmsg_output_t *output) { * Before random, fd and the locks below, all of which the * compressor thread uses. */ - async_res = _output_nmsg_async_destroy(*output); + async_res = _output_async_destroy(*output); if (res == nmsg_res_success) res = async_res; @@ -316,6 +316,7 @@ nmsg_output_close(nmsg_output_t *output) { close((*output)->stream->fd); } nmsg_container_destroy(&(*output)->stream->c); + pthread_cond_destroy(&(*output)->stream->c_drained); pthread_mutex_destroy(&(*output)->stream->c_lock); pthread_mutex_destroy(&(*output)->stream->w_lock); free((*output)->stream); @@ -418,11 +419,6 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { output->stream->do_zlib = zlibout; } -void -nmsg_output_set_zlib_async(nmsg_output_t output, bool async) { - nmsg_output_set_zlib_workers(output, async ? 1 : 0); -} - void nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { nmsg_res res; @@ -454,12 +450,12 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { * strand an error a worker had recorded. */ if (workers > 0) { - res = _output_nmsg_async_init(output, workers); + res = _output_async_init(output, workers); if (res != nmsg_res_success) _nmsg_dprintf(1, "%s: could not start compressor: %s\n", __func__, nmsg_res_lookup(res)); } else { - res = _output_nmsg_async_destroy(output); + res = _output_async_destroy(output); if (res != nmsg_res_success) _nmsg_dprintf(1, "%s: compressor reported: %s\n", __func__, nmsg_res_lookup(res)); @@ -603,6 +599,7 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { pthread_mutex_init(&output->stream->c_lock, NULL); pthread_mutex_init(&output->stream->w_lock, NULL); + pthread_cond_init(&output->stream->c_drained, NULL); /* enable container sequencing */ if (output->stream->type == nmsg_stream_type_sock || @@ -627,6 +624,7 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { output->stream->c = nmsg_container_init(bufsz); if (output->stream->c == NULL) { nmsg_random_destroy(&output->stream->random); + pthread_cond_destroy(&output->stream->c_drained); pthread_mutex_destroy(&output->stream->c_lock); pthread_mutex_destroy(&output->stream->w_lock); free(output->stream); diff --git a/nmsg/output.h b/nmsg/output.h index 16e80829..eeec8a68 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -382,29 +382,6 @@ nmsg_output_set_group(nmsg_output_t output, unsigned group); void nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); -/** - * Compress and write containers on a dedicated thread instead of on the thread - * calling nmsg_output_write(). Only affects file outputs. - * - * One container may be in flight at a time. A writer that finishes a container - * while the previous one is still being compressed blocks until it completes. - * - * Because a container is written after nmsg_output_write() returns, a write - * error is reported by a later nmsg_output_write(), or by nmsg_output_flush() - * or nmsg_output_close(), rather than by the call that supplied the data. - * - * Call before the first write. Enabling this later is harmless, but disabling - * it after writing discards any error already recorded for a container that - * has not yet been reported. - * - * \param[in] output nmsg_output_t object. - * - * \param[in] async True (compress on a separate thread) or false (compress - * inline, the default). - */ -void -nmsg_output_set_zlib_async(nmsg_output_t output, bool async); - /** * Compress containers on a pool of worker threads instead of on the thread * that filled them. @@ -418,13 +395,25 @@ nmsg_output_set_zlib_async(nmsg_output_t output, bool async); * Writes stay in the order the containers were filled, so the output is * byte-identical whatever \a workers is set to. * - * File outputs only, and only when buffered. Call before the first write. + * \a workers is a ceiling rather than an allocation: threads are started as + * load calls for them, so an output that never saturates never pays for them. + * The count is capped at an internal limit. + * + * Because a container is written after nmsg_output_write() returns, a write + * error is reported by a later nmsg_output_write(), or by nmsg_output_flush() + * or nmsg_output_close(), rather than by the call that supplied the data. + * + * File outputs only, and only when buffered. + * + * Not thread-safe against a concurrent nmsg_output_write() on the same output: + * call it before the first write, or while no write is in flight. Under + * nmsg_io that is guaranteed, since a close event excludes writers. * * \param[in] output nmsg_output_t object. * - * \param[in] workers Number of compressor threads, or 0 to compress inline - * (the default). More than one is only useful when a single output is - * offered more data than one core can compress. + * \param[in] workers Maximum number of compressor threads, or 0 to compress + * inline (the default). More than one is only useful when a single output + * is offered more data than one core can compress. */ void nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); diff --git a/nmsg/output_async.c b/nmsg/output_async.c new file mode 100644 index 00000000..cb9dd77f --- /dev/null +++ b/nmsg/output_async.c @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The compressor pool for a stream output. + * + * Everything here is private to this unit: the ring, its slot states and the + * threads that walk it. output_nmsg.c hands a sealed container over with + * _output_async_submit() and gets the bytes written for it, and calls back + * into that file for the actual compressing and writing. + */ + +/* Import. */ + +#include "private.h" + +/* Data structures. */ + +/* + * A compressor pool: a ring of slots, up to nworkers compressor threads and + * one committer thread. + * + * A container is given a ticket when it is sealed, under c_lock, so tickets + * follow the order the containers were closed in. Slot i serves every ticket + * with (ticket % depth) == i, so the producer of ticket T waits only for + * ticket T - depth to have been written. + * + * Compression runs on whichever thread is free. Only the committer writes, and + * only in ticket order, so the byte stream is identical to the synchronous + * path however many workers are running. + * + * A producer that finds every worker busy compresses the container itself + * rather than waiting. That is exactly what the synchronous path does, so the + * pool is never slower than no pool at all. + * + * Workers are spawned on demand rather than up front, so 'nworkers' is a + * ceiling and not an allocation. Once spawned a worker lives until the pool is destroyed. + */ + +/* + * Ring size, independent of the worker ceiling. A slot itself is tiny; what it + * costs is the container it points at while a ticket is in flight, so the depth + * bounds worst-case backlog rather than resident memory. The margin keeps slots + * available for producers to deposit into while every worker is busy, and + * capping the ceiling at depth - margin stops the pool having more compressors + * than places to put their output. + */ +#define ASYNC_RING_DEPTH 32 +#define ASYNC_RING_MARGIN 8 +#define ASYNC_MAX_WORKERS (ASYNC_RING_DEPTH - ASYNC_RING_MARGIN) + +typedef enum { + slot_empty = 0, /* Free. */ + slot_work, /* Container waiting for a compressor. */ + slot_taken, /* A worker or a producer is compressing it. */ + slot_done, /* Compressed, waiting its turn to be written. */ + slot_frag /* Oversized: the committer must fragment it itself. */ +} async_slot_state; + +struct async_slot { + async_slot_state state; + nmsg_container_t co; /* slot_work, slot_frag */ + uint8_t *buf; /* slot_done */ + size_t buf_len; + nmsg_res res; +}; + +struct nmsg_ostr_async { + pthread_mutex_t lock; + pthread_cond_t work_ready; /* A slot became slot_work. */ + pthread_cond_t commit_ready; /* The committer's slot is ready. */ + pthread_cond_t slot_free; /* A slot became slot_empty. */ + struct async_slot *slots; + unsigned depth; + unsigned nworkers; /* Ceiling; workers start on demand. */ + unsigned nstarted; /* Workers that exist and must be joined. */ + unsigned busy; /* Containers assigned to workers. */ + uint64_t issued; /* Highest ticket claimed, plus one. */ + uint64_t commit_next; /* Ticket allowed to write now. */ + bool shutdown; + bool started; /* Committer exists; must be joined. */ + bool failed; /* No committer; stay inline. */ + bool spawn_failed; /* Worker spawn failed; logged once. */ + pthread_t *workers; + pthread_t committer; + nmsg_res first_error; /* Sticky; surfaced by flush. */ + nmsg_output_t output; + uint64_t n_inline; /* Containers a producer compressed. */ + uint64_t n_waited; /* Producers that waited for a slot. */ +}; + +/* + * Take a reference to the stream's pool, or return NULL if there is none or it + * is being torn down. Caller holds c_lock. + */ +struct nmsg_ostr_async * +_output_async_ref(struct nmsg_stream_output *ostr) +{ + if (ostr->so_pool == NULL || ostr->so_pool_closing) + return (NULL); + + ostr->so_inflight++; + + return (ostr->so_pool); +} + +/* Drop a reference taken by _output_async_ref() and wake any waiting teardown. */ +void +_output_async_unref(struct nmsg_stream_output *ostr) +{ + pthread_mutex_lock(&ostr->c_lock); + assert(ostr->so_inflight > 0); + if (--ostr->so_inflight == 0) + pthread_cond_broadcast(&ostr->c_drained); + pthread_mutex_unlock(&ostr->c_lock); +} + +nmsg_res +_output_async_init(nmsg_output_t output, unsigned nworkers) { + struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool; + nmsg_res res, old_res = nmsg_res_success; + + if (nworkers == 0) + return (nmsg_res_success); + + if (nworkers > ASYNC_MAX_WORKERS) + nworkers = ASYNC_MAX_WORKERS; + + /* + * An existing pool's ceiling cannot be changed in place, since its + * worker array and ring are already sized, so a different count means + * building a replacement. Do that before tearing the old one down: if + * the allocation fails there is still a working pool to keep. + */ + if (ostr->so_pool != NULL && ostr->so_pool->nworkers == nworkers) + return (nmsg_res_success); + + pool = calloc(1, sizeof(*pool)); + if (pool == NULL) + return (nmsg_res_memfail); + + res = nmsg_res_memfail; + + pool->slots = calloc(ASYNC_RING_DEPTH, sizeof(*pool->slots)); + if (pool->slots == NULL) + goto fail_slots; + + pool->workers = calloc(nworkers, sizeof(*pool->workers)); + if (pool->workers == NULL) + goto fail_workers; + + res = nmsg_res_failure; + + if (pthread_mutex_init(&pool->lock, NULL) != 0) + goto fail_mutex; + if (pthread_cond_init(&pool->work_ready, NULL) != 0) + goto fail_work_ready; + if (pthread_cond_init(&pool->commit_ready, NULL) != 0) + goto fail_commit_ready; + if (pthread_cond_init(&pool->slot_free, NULL) != 0) + goto fail_slot_free; + + pool->depth = ASYNC_RING_DEPTH; + pool->nworkers = nworkers; + pool->output = output; + + if (ostr->so_pool != NULL) + old_res = _output_async_destroy(output); + + pthread_mutex_lock(&ostr->c_lock); + + /* + * Tickets count for the life of the stream, not the life of the pool, + * so a pool built after the first write must start where the stream has + * got to. Seeded under c_lock, and the pool is published in the same + * hold, so a ticket either predates the pool and is compressed inline or + * belongs to it and is at or above commit_next -- never below, where the + * committer would wait for it forever. + */ + pool->commit_next = pool->issued = ostr->so_ticket; + + /* + * Carry any error the previous pool recorded but had not yet reported, + * so replacing a pool does not swallow a failed write. + */ + pool->first_error = old_res; + + ostr->so_pool = pool; + pthread_mutex_unlock(&ostr->c_lock); + + return (nmsg_res_success); + +fail_slot_free: + pthread_cond_destroy(&pool->commit_ready); +fail_commit_ready: + pthread_cond_destroy(&pool->work_ready); +fail_work_ready: + pthread_mutex_destroy(&pool->lock); +fail_mutex: + free(pool->workers); +fail_workers: + free(pool->slots); +fail_slots: + free(pool); + + /* Whatever pool was already in place is untouched and still running. */ + return (res); +} + +/* + * Stop the pool and reclaim it. Everything still queued is written first. + * Must run before the stream's fd, random and locks go away, since the threads + * use all of them. + */ +nmsg_res +_output_async_destroy(nmsg_output_t output) { + struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool = ostr->so_pool; + nmsg_res res; + bool started; + unsigned i, nstarted; + + if (pool == NULL) + return (nmsg_res_success); + + /* + * Stop handing tickets to the pool, then wait for the ones already + * handed out to arrive. Both happen under c_lock, which is what orders + * them against ticket issuance: once this returns, no producer is still + * on its way here holding a ticket the committer will wait for. + */ + pthread_mutex_lock(&ostr->c_lock); + ostr->so_pool_closing = true; + while (ostr->so_inflight > 0) + pthread_cond_wait(&ostr->c_drained, &ostr->c_lock); + pthread_mutex_unlock(&ostr->c_lock); + + pthread_mutex_lock(&pool->lock); + pool->shutdown = true; /* Set under the lock: a thread about */ + started = pool->started; /* to wait would miss the wakeup. */ + nstarted = pool->nstarted; + pthread_cond_broadcast(&pool->work_ready); + pthread_cond_broadcast(&pool->commit_ready); + pthread_cond_broadcast(&pool->slot_free); + pthread_mutex_unlock(&pool->lock); + + /* + * Workers and committer are joined on their own counters. + */ + for (i = 0; i < nstarted; i++) + pthread_join(pool->workers[i], NULL); + if (started) + pthread_join(pool->committer, NULL); + + if (pool->n_inline > 0 || pool->n_waited > 0) + _nmsg_dprintf(2, "%s: %u of %u worker(s) started; %" PRIu64 + " container(s) compressed by the reader, %" PRIu64 + " wait(s) for a free slot\n", __func__, nstarted, + pool->nworkers, pool->n_inline, pool->n_waited); + + /* Read after the joins; the threads write it until they exit. */ + res = pool->first_error; + + pthread_mutex_lock(&ostr->c_lock); + ostr->so_pool = NULL; + ostr->so_pool_closing = false; + pthread_mutex_unlock(&ostr->c_lock); + + pthread_cond_destroy(&pool->slot_free); + pthread_cond_destroy(&pool->commit_ready); + pthread_cond_destroy(&pool->work_ready); + pthread_mutex_destroy(&pool->lock); + free(pool->workers); + free(pool->slots); + free(pool); + + return (res); +} + +/* Note an error, keeping the first one seen. Caller holds pool->lock. */ +static void +async_record_error(struct nmsg_ostr_async *pool, nmsg_res res) +{ + if (res != nmsg_res_success && pool->first_error == nmsg_res_success) + pool->first_error = res; +} + +/* + * Compressor thread. Takes any slot that needs compressing, in whatever order + * they become ready: compression order does not matter, only write order does, + * and the committer enforces that. + */ +static void * +async_worker(void *arg) +{ + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + + pthread_mutex_lock(&pool->lock); + + for (;;) { + struct async_slot *slot = NULL; + nmsg_container_t co; + uint8_t *buf; + size_t buf_len; + nmsg_res res; + unsigned i; + + for (i = 0; i < pool->depth; i++) { + if (pool->slots[i].state == slot_work) { + slot = &pool->slots[i]; + break; + } + } + + if (slot == NULL) { + /* + * Nothing to compress. Exit only once the producers + * have stopped, so a container queued just before + * shutdown is still compressed and written; that + * happens on every clean SIGTERM. + */ + if (pool->shutdown) + break; + pthread_cond_wait(&pool->work_ready, &pool->lock); + continue; + } + + slot->state = slot_taken; + co = slot->co; + slot->co = NULL; + pthread_mutex_unlock(&pool->lock); + + res = _output_nmsg_container_compress(pool->output, &co, &buf, &buf_len); + + pthread_mutex_lock(&pool->lock); + pool->busy--; /* Counted at deposit; see container_submit(). */ + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); + } + + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * The only thread that writes. It takes tickets strictly in order, so the file + * is byte-identical to what the synchronous path would have produced no matter + * how many workers compressed in parallel. + * + * It is also the only caller of _output_nmsg_frag_write(). + */ +static void * +async_committer(void *arg) +{ + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + + pthread_mutex_lock(&pool->lock); + + for (;;) { + struct async_slot *slot = &pool->slots[pool->commit_next % pool->depth]; + uint8_t *buf; + size_t buf_len; + nmsg_container_t co; + nmsg_res res = nmsg_res_success; + + switch (slot->state) { + case slot_done: + buf = slot->buf; + buf_len = slot->buf_len; + res = slot->res; + slot->buf = NULL; + pthread_mutex_unlock(&pool->lock); + + /* + * _output_nmsg_send_buffer() frees buf. It is not called when the + * compression failed, but then buf is NULL anyway; see + * _output_nmsg_container_compress(). + */ + if (res == nmsg_res_success) + res = _output_nmsg_send_buffer(pool->output, buf, buf_len); + + pthread_mutex_lock(&pool->lock); + break; + + case slot_frag: + co = slot->co; + slot->co = NULL; + pthread_mutex_unlock(&pool->lock); + + /* _output_nmsg_frag_write() takes the container by value and destroys it. */ + res = _output_nmsg_frag_write(pool->output, co); + + pthread_mutex_lock(&pool->lock); + break; + + case slot_empty: + case slot_work: + case slot_taken: + /* + * The next ticket is not ready. Exit only when the + * producers have stopped and every ticket they issued + * has been written. + */ + if (pool->shutdown && pool->commit_next >= pool->issued) + goto out; + pthread_cond_wait(&pool->commit_ready, &pool->lock); + continue; + } + + async_record_error(pool, res); + slot->state = slot_empty; + pool->commit_next++; + pthread_cond_broadcast(&pool->slot_free); + } + +out: + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * Start the committer. Called under pool->lock on first submit rather than when + * the output is configured, because nmsgtool creates its outputs before it + * daemonizes, and daemonize() is a bare fork() which no thread survives. + * Starting on first write puts the threads in whichever process does the + * writing. + * + * Only the committer starts here. Workers are added by async_spawn_worker() as + * load calls for them. + */ +static void +async_start(struct nmsg_ostr_async *pool) +{ + int pthread_res; + + pthread_res = pthread_create(&pool->committer, NULL, async_committer, pool); + if (pthread_res != 0) { + /* + * Nothing can be written without a committer, so give up on the + * pool entirely and compress inline from here on. No worker has + * been created yet and no container has been deposited, so there + * is nothing to unwind. + */ + pool->failed = true; + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, + strerror(pthread_res)); + return; + } + + pool->started = true; +} + +/* + * Add a compressor thread, up to the ceiling. Called under pool->lock when a + * producer finds every existing worker busy, so the pool grows to the load it + * actually sees instead of to the configured ceiling. + * + * Returns false if the thread could not be created, which is not fatal: the + * caller compresses that container itself and the pool keeps running with the + * workers it has. + */ +static bool +async_spawn_worker(struct nmsg_ostr_async *pool) +{ + int pthread_res; + + pthread_res = pthread_create(&pool->workers[pool->nstarted], NULL, + async_worker, pool); + if (pthread_res != 0) { + /* Logged once; a persistent failure would flood the log. */ + if (!pool->spawn_failed) { + pool->spawn_failed = true; + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", + __func__, strerror(pthread_res)); + } + return (false); + } + + pool->nstarted++; + + return (true); +} + +/* + * Wait until every ticket issued so far has been written, and take any error + * the pool recorded. Under nmsg_io this cannot starve: check_close_event() + * holds io_output->refcount across the write and call_close_fp() waits for it + * to drop, so no other thread is inside nmsg_output_write() while a close runs. + * A caller driving nmsg_output_flush() directly from several threads has no + * such guarantee. + */ +nmsg_res +_output_async_drain(struct nmsg_ostr_async *pool) +{ + nmsg_res res; + + if (pool == NULL) + return (nmsg_res_success); + + pthread_mutex_lock(&pool->lock); + while (!pool->failed && pool->commit_next < pool->issued) + pthread_cond_wait(&pool->slot_free, &pool->lock); + res = pool->first_error; + pool->first_error = nmsg_res_success; + pthread_mutex_unlock(&pool->lock); + + return (res); +} + +/* + * Hand a sealed container to the pool, consuming it only if the pool takes it. + * Returns false if it does not, and the caller writes the container itself. + * The pool reference is released either way. + * + * On success *res_out carries the error from an EARLIER container, since the + * one just handed over has not been written yet. + */ +bool +_output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, + nmsg_container_t *co, bool is_frag, uint64_t ticket, + nmsg_res *res_out) +{ + struct nmsg_stream_output *ostr = output->stream; + struct async_slot *slot; + nmsg_res res = nmsg_res_success; + uint8_t *buf; + size_t buf_len; + bool inline_compress = false; + + pthread_mutex_lock(&pool->lock); + + if (!pool->started && !pool->failed && !pool->shutdown) + async_start(pool); + + /* + * Only reachable when the committer could not be started: teardown drains + * outstanding tickets before setting shutdown, so a ticket that got this + * far still has a pool to go to. + */ + if (!pool->started || pool->failed || pool->shutdown) { + pthread_mutex_unlock(&pool->lock); + _output_async_unref(ostr); + return (false); + } + + slot = &pool->slots[ticket % pool->depth]; + + /* + * Wait for the slot this ticket owns, which the ticket 'depth' earlier + * releases when it is written. Only reached when the writer has fallen + * a whole ring behind. + */ + if (ticket >= pool->commit_next + pool->depth) { + pool->n_waited++; + while (ticket >= pool->commit_next + pool->depth && !pool->shutdown) + pthread_cond_wait(&pool->slot_free, &pool->lock); + } + + assert(slot->state == slot_empty); + + if (ticket >= pool->issued) + pool->issued = ticket + 1; + + if (is_frag) { + /* Only the committer fragments; see async_committer(). */ + slot->co = *co; + *co = NULL; + slot->state = slot_frag; + pthread_cond_broadcast(&pool->commit_ready); + } else if (pool->busy < pool->nstarted || + (pool->nstarted < pool->nworkers && async_spawn_worker(pool))) + { + /* + * A worker is free, or the ceiling left room to add one: hand + * the container over and get back to reading. + */ + slot->co = *co; + *co = NULL; + slot->state = slot_work; + pool->busy++; + pthread_cond_signal(&pool->work_ready); + } else { + /* + * Every worker is busy and the ceiling is reached. Compress it + * here rather than wait, then deposit the result and return. + */ + slot->state = slot_taken; + pool->n_inline++; + inline_compress = true; + } + + res = pool->first_error; + pool->first_error = nmsg_res_success; + pthread_mutex_unlock(&pool->lock); + + if (inline_compress) { + nmsg_res c_res = _output_nmsg_container_compress(output, co, &buf, &buf_len); + + pthread_mutex_lock(&pool->lock); + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = c_res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); + pthread_mutex_unlock(&pool->lock); + } + + *res_out = res; + _output_async_unref(ostr); + + return (true); +} diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index a9e8a370..cb8b2065 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -21,193 +21,20 @@ /* Forward. */ static nmsg_res container_write(nmsg_output_t, nmsg_container_t*); -static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool, uint64_t); -static nmsg_res frag_write(nmsg_output_t, nmsg_container_t); -static nmsg_res send_buffer(nmsg_output_t, uint8_t *buf, size_t len); -static nmsg_res async_drain(struct nmsg_ostr_async *); +static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool, uint64_t, + struct nmsg_ostr_async *); /* Data structures. */ -/* - * A compressor pool: a ring of slots, nworkers compressor threads and one - * committer thread. - * - * A container is given a ticket when it is sealed, under c_lock, so tickets - * follow the order the containers were closed in. Slot i serves every ticket - * with (ticket % depth) == i, so the producer of ticket T waits only for - * ticket T - depth to have been written. - * - * Compression runs on whichever thread is free. Only the committer writes, and - * only in ticket order, so the byte stream is identical to the synchronous - * path however many workers are running. - * - * A producer that finds every worker busy compresses the container itself - * rather than waiting. That is exactly what the synchronous path does, so the - * pool is never slower than no pool at all. - */ -typedef enum { - slot_empty = 0, /* Free. */ - slot_work, /* Container waiting for a compressor. */ - slot_taken, /* A worker or a producer is compressing it. */ - slot_done, /* Compressed, waiting its turn to be written. */ - slot_frag /* Oversized: the committer must fragment it itself. */ -} async_slot_state; - -struct async_slot { - async_slot_state state; - nmsg_container_t co; /* slot_work, slot_frag */ - uint8_t *buf; /* slot_done */ - size_t buf_len; - nmsg_res res; -}; - -struct nmsg_ostr_async { - pthread_mutex_t lock; - pthread_cond_t work_ready; /* A slot became slot_work. */ - pthread_cond_t commit_ready; /* The committer's slot is ready. */ - pthread_cond_t slot_free; /* A slot became slot_empty. */ - struct async_slot *slots; - unsigned depth; - unsigned nworkers; - unsigned busy; /* Workers currently compressing. */ - uint64_t issued; /* Highest ticket claimed, plus one. */ - uint64_t commit_next; /* Ticket allowed to write now. */ - bool shutdown; - bool started; - bool failed; /* Thread creation failed; stay inline. */ - pthread_t *workers; - pthread_t committer; - nmsg_res first_error; /* Sticky; surfaced by flush. */ - nmsg_output_t output; - uint64_t n_inline; /* Containers a producer compressed. */ - uint64_t n_waited; /* Producers that waited for a slot. */ -}; /* Internal functions. */ -nmsg_res -_output_nmsg_async_init(nmsg_output_t output, unsigned nworkers) { - struct nmsg_ostr_async *pool; - unsigned depth; - - if (nworkers == 0) - return (nmsg_res_success); - - if (output->stream->so_pool != NULL) - return (nmsg_res_success); - - /* - * One slot per worker to compress in, plus a fixed margin for - * producers to deposit into and for finished buffers waiting their - * turn to be written. The margin does not need to scale with the - * worker count: a producer that cannot get a slot compresses the - * container itself rather than waiting, so a tight ring costs a little - * of the parallelism and never blocks a reader. - * - * A slot holds a container's payloads in memory, which is not the - * 1 MiB serialized size -- a channel of 30-byte payloads puts ~37k of - * them in a container, over 5 MB. That is what bounds the ring, and - * why the automatic worker count is capped. - */ - depth = nworkers + 8; - - pool = calloc(1, sizeof(*pool)); - if (pool == NULL) - return (nmsg_res_memfail); - - pool->slots = calloc(depth, sizeof(*pool->slots)); - if (pool->slots == NULL) - goto fail_slots; - - pool->workers = calloc(nworkers, sizeof(*pool->workers)); - if (pool->workers == NULL) - goto fail_workers; - - if (pthread_mutex_init(&pool->lock, NULL) != 0) - goto fail_mutex; - if (pthread_cond_init(&pool->work_ready, NULL) != 0) - goto fail_work_ready; - if (pthread_cond_init(&pool->commit_ready, NULL) != 0) - goto fail_commit_ready; - if (pthread_cond_init(&pool->slot_free, NULL) != 0) - goto fail_slot_free; - - pool->depth = depth; - pool->nworkers = nworkers; - pool->output = output; - output->stream->so_pool = pool; - - return (nmsg_res_success); - -fail_slot_free: - pthread_cond_destroy(&pool->commit_ready); -fail_commit_ready: - pthread_cond_destroy(&pool->work_ready); -fail_work_ready: - pthread_mutex_destroy(&pool->lock); -fail_mutex: - free(pool->workers); -fail_workers: - free(pool->slots); -fail_slots: - free(pool); - - return (nmsg_res_failure); -} - -/* - * Stop the pool and reclaim it. Everything still queued is written first. - * Must run before the stream's fd, random and locks go away, since the threads - * use all of them. - */ -nmsg_res -_output_nmsg_async_destroy(nmsg_output_t output) { - struct nmsg_ostr_async *pool = output->stream->so_pool; - nmsg_res res; - bool started; - unsigned i; - - if (pool == NULL) - return (nmsg_res_success); - - pthread_mutex_lock(&pool->lock); - pool->shutdown = true; /* Set under the lock: a thread about */ - started = pool->started; /* to wait would miss the wakeup. */ - pthread_cond_broadcast(&pool->work_ready); - pthread_cond_broadcast(&pool->commit_ready); - pthread_cond_broadcast(&pool->slot_free); - pthread_mutex_unlock(&pool->lock); - - if (started) { - for (i = 0; i < pool->nworkers; i++) - pthread_join(pool->workers[i], NULL); - pthread_join(pool->committer, NULL); - } - - if (pool->n_inline > 0 || pool->n_waited > 0) - _nmsg_dprintf(2, "%s: %u worker(s); %" PRIu64 " container(s) " - "compressed by the reader, %" PRIu64 " wait(s) for " - "a free slot\n", __func__, pool->nworkers, - pool->n_inline, pool->n_waited); - - /* Read after the joins; the threads write it until they exit. */ - res = pool->first_error; - - pthread_cond_destroy(&pool->slot_free); - pthread_cond_destroy(&pool->commit_ready); - pthread_cond_destroy(&pool->work_ready); - pthread_mutex_destroy(&pool->lock); - free(pool->workers); - free(pool->slots); - free(pool); - output->stream->so_pool = NULL; - return (res); -} nmsg_res _output_nmsg_flush(nmsg_output_t output) { struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool, *submit_pool = NULL; nmsg_res res = nmsg_res_success; nmsg_res drain_res; nmsg_container_t old_c = NULL; @@ -215,9 +42,16 @@ _output_nmsg_flush(nmsg_output_t output) { pthread_mutex_lock(&ostr->c_lock); + /* + * The pool is picked up here rather than read again later, so a teardown + * running alongside this flush cannot free it underneath. + */ + pool = _output_async_ref(ostr); + if (nmsg_container_get_num_payloads(ostr->c) > 0) { old_c = ostr->c; ticket = ostr->so_ticket++; + submit_pool = _output_async_ref(ostr); ostr->c = nmsg_container_init(ostr->bufsz); if (ostr->c == NULL) @@ -236,7 +70,7 @@ _output_nmsg_flush(nmsg_output_t output) { if (old_c != NULL) { nmsg_res sub_res; - sub_res = container_submit(output, &old_c, false, ticket); + sub_res = container_submit(output, &old_c, false, ticket, submit_pool); if (res == nmsg_res_success) res = sub_res; } @@ -245,10 +79,13 @@ _output_nmsg_flush(nmsg_output_t output) { * A flush means the data has been written, so wait out anything the * pool is still holding. */ - drain_res = async_drain(ostr->so_pool); + drain_res = _output_async_drain(pool); if (res == nmsg_res_success) res = drain_res; + if (pool != NULL) + _output_async_unref(ostr); + return (res); } @@ -256,9 +93,10 @@ nmsg_res _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { Nmsg__NmsgPayload *np; struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool; nmsg_container_t old_c, new_c; nmsg_res res; - uint64_t ticket = 0; + uint64_t ticket; bool must_flush, is_buffered; assert(msg->np != NULL); @@ -281,6 +119,8 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { retry: must_flush = false; old_c = new_c = NULL; + pool = NULL; + ticket = 0; pthread_mutex_lock(&ostr->c_lock); /* Lock for add to container. */ @@ -321,9 +161,12 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { * container's contents become final. Taking it in * container_submit() would order the containers by which * thread won the race after the unlock, which is not the - * order they were filled in. + * order they were filled in. The pool is claimed in the same + * hold, so the ticket and the pool that will serve it are + * chosen together. */ ticket = ostr->so_ticket++; + pool = _output_async_ref(ostr); } pthread_mutex_unlock(&ostr->c_lock); /* Release locked container to other threads. */ @@ -333,16 +176,16 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ - res = container_submit(output, &old_c, false, ticket); /* Write data from prior container. */ + res = container_submit(output, &old_c, false, ticket, pool); /* Write data from prior container. */ if (res != nmsg_res_success) return (res); /* Proceed to write current message to new container. */ goto retry; } else if (res == nmsg_res_success && is_buffered == false) { /* Includes current message. */ - res = container_submit(output, &old_c, false, ticket); + res = container_submit(output, &old_c, false, ticket, pool); } else if (res == nmsg_res_container_overfull) { /* Includes current message. */ - res = container_submit(output, &old_c, true, ticket); + res = container_submit(output, &old_c, true, ticket, pool); } return (res); @@ -354,8 +197,8 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { * Compress one container into a standalone buffer. The container is destroyed * either way, as container_write() does. */ -static nmsg_res -container_compress(nmsg_output_t output, nmsg_container_t *co, +nmsg_res +_output_nmsg_container_compress(nmsg_output_t output, nmsg_container_t *co, uint8_t **buf, size_t *buf_len) { struct nmsg_stream_output *ostr = output->stream; @@ -363,7 +206,10 @@ container_compress(nmsg_output_t output, nmsg_container_t *co, uint32_t seq; uint8_t *shrunk; - /* Multiple threads can enter here at once. */ + /* + * Multiple threads can enter here at once, so the numbers are handed out + * in compression order rather than write order. + */ seq = atomic_fetch_add_explicit(&ostr->so_sequence_num, 1, memory_order_relaxed); res = nmsg_container_serialize(*co, buf, buf_len, true, /* do_header */ @@ -399,323 +245,55 @@ container_write(nmsg_output_t output, nmsg_container_t *co) size_t buf_len; uint8_t *buf; - res = container_compress(output, co, &buf, &buf_len); + res = _output_nmsg_container_compress(output, co, &buf, &buf_len); if (res != nmsg_res_success) return (res); - return (send_buffer(output, buf, buf_len)); -} - -/* Note an error, keeping the first one seen. Caller holds pool->lock. */ -static void -async_record_error(struct nmsg_ostr_async *pool, nmsg_res res) -{ - if (res != nmsg_res_success && pool->first_error == nmsg_res_success) - pool->first_error = res; -} - -/* - * Compressor thread. Takes any slot that needs compressing, in whatever order - * they become ready: compression order does not matter, only write order does, - * and the committer enforces that. - */ -static void * -async_worker(void *arg) -{ - struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; - - pthread_mutex_lock(&pool->lock); - - for (;;) { - struct async_slot *slot = NULL; - nmsg_container_t co; - uint8_t *buf; - size_t buf_len; - nmsg_res res; - unsigned i; - - for (i = 0; i < pool->depth; i++) { - if (pool->slots[i].state == slot_work) { - slot = &pool->slots[i]; - break; - } - } - - if (slot == NULL) { - /* - * Nothing to compress. Exit only once the producers - * have stopped, so a container queued just before - * shutdown is still compressed and written; that - * happens on every clean SIGTERM. - */ - if (pool->shutdown) - break; - pthread_cond_wait(&pool->work_ready, &pool->lock); - continue; - } - - slot->state = slot_taken; - co = slot->co; - slot->co = NULL; - pool->busy++; - pthread_mutex_unlock(&pool->lock); - - res = container_compress(pool->output, &co, &buf, &buf_len); - - pthread_mutex_lock(&pool->lock); - pool->busy--; - slot->buf = buf; - slot->buf_len = buf_len; - slot->res = res; - slot->state = slot_done; - pthread_cond_broadcast(&pool->commit_ready); - } - - pthread_mutex_unlock(&pool->lock); - - return (NULL); + return (_output_nmsg_send_buffer(output, buf, buf_len)); } -/* - * The only thread that writes. It takes tickets strictly in order, so the file - * is byte-identical to what the synchronous path would have produced no matter - * how many workers compressed in parallel. - * - * It is also the only caller of frag_write(). - */ -static void * -async_committer(void *arg) -{ - struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; - - pthread_mutex_lock(&pool->lock); - - for (;;) { - struct async_slot *slot = &pool->slots[pool->commit_next % pool->depth]; - async_slot_state state = slot->state; - nmsg_res res; - - if (state == slot_done) { - uint8_t *buf = slot->buf; - size_t buf_len = slot->buf_len; - - res = slot->res; - slot->buf = NULL; - pthread_mutex_unlock(&pool->lock); - - /* send_buffer() owns buf from here, error or not. */ - if (res == nmsg_res_success) - res = send_buffer(pool->output, buf, buf_len); - - pthread_mutex_lock(&pool->lock); - } else if (state == slot_frag) { - nmsg_container_t co = slot->co; - - slot->co = NULL; - pthread_mutex_unlock(&pool->lock); - - /* frag_write() takes the container by value and destroys it. */ - res = frag_write(pool->output, co); - - pthread_mutex_lock(&pool->lock); - } else { - /* - * The next ticket is not ready. Exit only when the - * producers have stopped and every ticket they issued - * has been written. - */ - if (pool->shutdown && pool->commit_next >= pool->issued) - break; - pthread_cond_wait(&pool->commit_ready, &pool->lock); - continue; - } - - async_record_error(pool, res); - slot->state = slot_empty; - pool->commit_next++; - pthread_cond_broadcast(&pool->slot_free); - } - pthread_mutex_unlock(&pool->lock); - - return (NULL); -} - -/* - * Start the threads. Called under pool->lock on first submit rather than when - * the output is configured, because nmsgtool creates its outputs before it - * daemonizes, and daemonize() is a bare fork() which no thread survives. - * Starting on first write puts the threads in whichever process does the - * writing. - */ -static void -async_start(struct nmsg_ostr_async *pool) -{ - unsigned i; - int pthread_res; - for (i = 0; i < pool->nworkers; i++) { - pthread_res = pthread_create(&pool->workers[i], NULL, async_worker, pool); - if (pthread_res != 0) - goto fail; - } - pthread_res = pthread_create(&pool->committer, NULL, async_committer, pool); - if (pthread_res != 0) - goto fail; - pool->started = true; - return; -fail: - /* - * Stop whatever did start and fall back to compressing inline. Joining - * needs the lock released, and the caller still holds a container, so - * this stays simple: the threads created so far see shutdown and exit, - * and destroy joins them. - */ - pool->nworkers = i; - pool->failed = true; - pool->shutdown = true; - pool->started = (i > 0); - pthread_cond_broadcast(&pool->work_ready); - pthread_cond_broadcast(&pool->commit_ready); - _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, - strerror(pthread_res)); -} -/* - * Wait until every ticket issued so far has been written, and take any error - * the pool recorded. Under nmsg_io this cannot starve: check_close_event() - * holds io_output->refcount across the write and call_close_fp() waits for it - * to drop, so no other thread is inside nmsg_output_write() while a close runs. - * A caller driving nmsg_output_flush() directly from several threads has no - * such guarantee. - */ +/* Compress and write on the calling thread. The container is consumed. */ static nmsg_res -async_drain(struct nmsg_ostr_async *pool) +container_submit_inline(nmsg_output_t output, nmsg_container_t *co, bool is_frag) { - nmsg_res res; - - if (pool == NULL) - return (nmsg_res_success); + if (is_frag) { + nmsg_container_t tmp = *co; - pthread_mutex_lock(&pool->lock); - while (pool->started && !pool->failed && pool->commit_next < pool->issued) - pthread_cond_wait(&pool->slot_free, &pool->lock); - res = pool->first_error; - pool->first_error = nmsg_res_success; - pthread_mutex_unlock(&pool->lock); + /* + * _output_nmsg_frag_write() takes the container by value and destroys it + * internally, unlike container_write(). + */ + *co = NULL; + return (_output_nmsg_frag_write(output, tmp)); + } - return (res); + return (container_write(output, co)); } /* * Hand a finished container to the pool, or process it inline if there is no - * pool. The container is consumed either way. + * pool or the pool declines it. The container is consumed either way. * - * The pool reports success for the container it just took, since that - * container has not been written yet. An error from an EARLIER container is - * returned here. + * 'pool' is the reference taken under c_lock when the ticket was issued, so it + * is NULL exactly when the ticket was never promised to a pool. */ static nmsg_res container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag, - uint64_t ticket) + uint64_t ticket, struct nmsg_ostr_async *pool) { - struct nmsg_ostr_async *pool = output->stream->so_pool; - struct async_slot *slot; - nmsg_res res = nmsg_res_success; - uint8_t *buf; - size_t buf_len; - bool inline_compress = false; - - if (pool == NULL) - goto inline_path; - - pthread_mutex_lock(&pool->lock); - - if (!pool->started && !pool->failed && !pool->shutdown) - async_start(pool); - - if (!pool->started || pool->failed || pool->shutdown) { - pthread_mutex_unlock(&pool->lock); - goto inline_path; - } - - slot = &pool->slots[ticket % pool->depth]; - - /* - * Wait for the slot this ticket owns, which the ticket 'depth' earlier - * releases when it is written. Only reached when the writer has fallen - * a whole ring behind. - */ - if (slot->state != slot_empty) { - pool->n_waited++; - while (slot->state != slot_empty && !pool->shutdown) - pthread_cond_wait(&pool->slot_free, &pool->lock); - - if (pool->shutdown) { - pthread_mutex_unlock(&pool->lock); - goto inline_path; - } - } - - if (ticket >= pool->issued) - pool->issued = ticket + 1; - - if (is_frag) { - /* Only the committer fragments; see async_committer(). */ - slot->co = *co; - *co = NULL; - slot->state = slot_frag; - pthread_cond_broadcast(&pool->commit_ready); - } else if (pool->busy < pool->nworkers) { - /* A worker is free: hand it over and get back to reading. */ - slot->co = *co; - *co = NULL; - slot->state = slot_work; - pthread_cond_signal(&pool->work_ready); - } else { - /* - * Every worker is busy. Compress it here rather than wait then deposit - * the result and return. - */ - slot->state = slot_taken; - pool->n_inline++; - inline_compress = true; - } - - res = pool->first_error; - pool->first_error = nmsg_res_success; - pthread_mutex_unlock(&pool->lock); - - if (inline_compress) { - nmsg_res c_res = container_compress(output, co, &buf, &buf_len); - - pthread_mutex_lock(&pool->lock); - slot->buf = buf; - slot->buf_len = buf_len; - slot->res = c_res; - slot->state = slot_done; - pthread_cond_broadcast(&pool->commit_ready); - pthread_mutex_unlock(&pool->lock); - } - - return (res); - -inline_path: - if (is_frag) { - nmsg_container_t tmp = *co; + nmsg_res res; - /* - * frag_write() takes the container by value and destroys it - * internally, unlike container_write(). - */ - *co = NULL; - return (frag_write(output, tmp)); - } + if (pool != NULL && + _output_async_submit(pool, output, co, is_frag, ticket, &res)) + return (res); - return (container_write(output, co)); + return (container_submit_inline(output, co, is_frag)); } static nmsg_res @@ -808,8 +386,8 @@ write_file(int fd, uint8_t *buf, size_t len) * * Returns status of send. */ -static nmsg_res -send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) +nmsg_res +_output_nmsg_send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) { struct nmsg_stream_output *ostr = output->stream; nmsg_res res; @@ -907,8 +485,8 @@ header_serialize(uint8_t *buf, uint8_t flags, uint32_t len) store_net32(buf, len); } -static nmsg_res -frag_write(nmsg_output_t output, nmsg_container_t co) +nmsg_res +_output_nmsg_frag_write(nmsg_output_t output, nmsg_container_t co) { Nmsg__NmsgFragment nf; struct nmsg_stream_output *ostr = output->stream; @@ -944,7 +522,7 @@ frag_write(nmsg_output_t output, nmsg_container_t co) if (ostr->do_zlib && len <= max_fragsz) { /* write out the unfragmented NMSG container */ - res = send_buffer(output, packed, len); + res = _output_nmsg_send_buffer(output, packed, len); goto frag_out; } @@ -978,7 +556,7 @@ frag_write(nmsg_output_t output, nmsg_container_t co) fraglen += NMSG_HDRLSZ_V2; /* send the serialized fragment */ - res = send_buffer(output, frag_packed, fraglen); + res = _output_nmsg_send_buffer(output, frag_packed, fraglen); } free(packed); diff --git a/nmsg/private.h b/nmsg/private.h index 56854576..a05bbe95 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -337,6 +337,9 @@ struct nmsg_stream_output { atomic_uint_fast32_t so_sequence_num; uint64_t sequence_id; uint64_t so_ticket; /* Next container ticket; c_lock. */ + unsigned so_inflight; /* Tickets owed to the pool; c_lock. */ + bool so_pool_closing; /* Pool teardown started; c_lock. */ + pthread_cond_t c_drained; /* so_inflight == 0; c_lock. */ struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ }; @@ -593,8 +596,19 @@ nmsg_output_t _output_open_kafka(void *s, size_t bufsz); /* from output_nmsg.c */ nmsg_res _output_nmsg_flush(nmsg_output_t); nmsg_res _output_nmsg_write(nmsg_output_t, nmsg_message_t); -nmsg_res _output_nmsg_async_init(nmsg_output_t, unsigned); -nmsg_res _output_nmsg_async_destroy(nmsg_output_t); +nmsg_res _output_nmsg_container_compress(nmsg_output_t, nmsg_container_t *, + uint8_t **, size_t *); +nmsg_res _output_nmsg_send_buffer(nmsg_output_t, uint8_t *, size_t); +nmsg_res _output_nmsg_frag_write(nmsg_output_t, nmsg_container_t); + +/* from output_async.c */ +nmsg_res _output_async_init(nmsg_output_t, unsigned); +nmsg_res _output_async_destroy(nmsg_output_t); +nmsg_res _output_async_drain(struct nmsg_ostr_async *); +struct nmsg_ostr_async *_output_async_ref(struct nmsg_stream_output *); +void _output_async_unref(struct nmsg_stream_output *); +bool _output_async_submit(struct nmsg_ostr_async *, nmsg_output_t, + nmsg_container_t *, bool, uint64_t, nmsg_res *); #ifdef HAVE_LIBRDKAFKA nmsg_res _output_kafka_payload_write(nmsg_output_t, nmsg_message_t); nmsg_res _output_kafka_payload_flush(nmsg_output_t); diff --git a/src/io.c b/src/io.c index 4a602969..6b9748fb 100644 --- a/src/io.c +++ b/src/io.c @@ -598,6 +598,7 @@ add_file_output(nmsgtool_ctx *c, const char *fname) { fprintf(stderr, "%s: nmsg file output: %s\n", argv_program, fname); c->n_outputs += 1; + c->n_file_outputs += 1; } void diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 63587b9f..6b93db2d 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -20,6 +20,9 @@ #include #include #include +#ifdef __linux__ +#include +#endif /* __linux__ */ #include #include #include @@ -330,7 +333,7 @@ static argv_t args[] = { ARGV_INT, &ctx.zasync, "n", - "compress file output on n threads (-1 chooses)" }, + "compress file output on n threads" }, { ARGV_LAST, 0, 0, 0, 0, 0 } }; @@ -438,39 +441,61 @@ usage(const char *msg) { exit(msg == NULL ? EXIT_SUCCESS : EXIT_FAILURE); } +/* + * Cores this process may actually run on. + */ +long +nmsgtool_ncpu(void) { + long ncpu = -1; +#ifdef __linux__ + cpu_set_t set; + + if (sched_getaffinity(0, sizeof(set), &set) == 0) + ncpu = CPU_COUNT(&set); +#endif /* __linux__ */ + + if (ncpu < 1) + ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if (ncpu < 1) + ncpu = 1; + + return (ncpu); +} + /* * How many compressor threads an output should get. * * A negative --zasync means auto. Two things bound the answer: * - * Demand scales with the input sockets. Each reader thread can saturate - * roughly one core compressing, and covering that takes about two workers - * per socket. + * Demand scales with the inputs. Each reader thread can saturate roughly one + * core compressing, and covering that takes about two workers per input. * * Supply is the cores the readers leave. Workers beyond that only contend. * - * The floor matters because one socket can carry several cores' worth on its - * own, so a count that merely followed the socket count would under-serve + * The floor matters because one input can carry several cores' worth on its + * own, so a count that merely followed the input count would under-serve * exactly the case the pool exists for. * - * Getting it wrong is cheap in one direction: a reader that finds every worker - * busy compresses the container itself, so too small a pool degrades to the - * behaviour of no pool rather than stalling. Too large only wastes idle - * threads and ring slots, and a slot holds a container's payloads in memory. + * The budget is then split across the file outputs, since each gets its own + * pool and they share these same cores. Each output keeps at least one worker: + * one is still enough to take compression off the reader, which is the point. + * + * Getting it wrong is cheap in both directions. Too small a pool degrades to + * the behaviour of no pool rather than stalling, because a reader that finds + * every worker busy compresses the container itself. Too large only raises a + * ceiling that is never reached, since workers are started on demand. */ static unsigned zworkers_count(nmsgtool_ctx *c) { long ncpu; int n, spare; - if (c->zasync == 0) + if (c->zasync == 0 || c->n_file_outputs == 0) return (0); if (c->zasync > 0) return ((unsigned) c->zasync); - ncpu = sysconf(_SC_NPROCESSORS_ONLN); - if (ncpu < 1) - ncpu = 1; + ncpu = nmsgtool_ncpu(); n = 2 * c->n_inputs; @@ -480,12 +505,16 @@ zworkers_count(nmsgtool_ctx *c) { if (n < NMSGTOOL_ZWORKERS_MIN) n = NMSGTOOL_ZWORKERS_MIN; + n /= c->n_file_outputs; + if (n < 1) + n = 1; + return ((unsigned) n); } /* - * Resolve -W and apply it to the outputs that already exist. Deferred to the - * end of process_args() because the input count is not final until then: a + * Resolve --zasync and apply it to the outputs that already exist. Deferred to + * the end of process_args() because the input count is not final until then: a * channel alias (-C) expands to its sockets after the outputs have been * created, which is exactly the case the pool is sized for. */ @@ -504,7 +533,7 @@ setup_nmsg_output_workers(nmsgtool_ctx *c) { } if (c->zworkers_resolved > 0 && c->debug >= 2) - fprintf(stderr, "%s: compressing on %u thread(s)\n", + fprintf(stderr, "%s: compressing on up to %u thread(s) per output\n", argv_program, c->zworkers_resolved); } @@ -518,7 +547,7 @@ setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_operator(output, c->set_operator); nmsg_output_set_group(output, c->set_group); - /* Outputs made before -W is resolved; see setup_nmsg_output_workers(). */ + /* Outputs made before --zasync is resolved; see setup_nmsg_output_workers(). */ if (c->initial_outputs != NULL) output_vec_add(c->initial_outputs, output); } diff --git a/src/nmsgtool.h b/src/nmsgtool.h index 58a7c88f..ca590fd3 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -59,6 +59,13 @@ VECTOR_GENERATE(output_vec, nmsg_output_t) */ #define NMSGTOOL_ZWORKERS_MIN 4 +/* + * Ceiling for an explicit --zasync. Workers start on demand, so this only + * bounds how far a saturated output may grow; libnmsg applies its own limit + * on top, since it cannot assume its caller validated anything. + */ +#define NMSGTOOL_ZWORKERS_MAX(ncpu) (2 * (ncpu)) + typedef struct { /* parameters */ argv_array_t filters, statsmods; @@ -78,6 +85,7 @@ typedef struct { /* state */ char *endline_str; int n_inputs, n_outputs; + int n_file_outputs; /* Outputs a compressor pool can serve. */ unsigned zworkers_resolved; /* 0 until process_args() finishes. */ output_vec *initial_outputs; /* Non-NULL only during process_args(). */ nmsg_io_t io; @@ -148,6 +156,7 @@ void add_zsock_input(nmsgtool_ctx *, const char *); void add_zsock_output(nmsgtool_ctx *, const char *); void add_filter_module(nmsgtool_ctx *, const char *); void add_stats_module(nmsgtool_ctx *, const char *); +long nmsgtool_ncpu(void); void pidfile_write(FILE *); void process_args(nmsgtool_ctx *); void setup_nmsg_input(nmsgtool_ctx *, nmsg_input_t); diff --git a/src/process_args.c b/src/process_args.c index e8c32afe..6f94c203 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -125,6 +125,10 @@ process_args(nmsgtool_ctx *c) { if (c->mtu == 0) c->mtu = NMSG_WBUFSZ_JUMBO; + if (c->zasync < -1 || c->zasync > NMSGTOOL_ZWORKERS_MAX(nmsgtool_ncpu())) + usage("--zasync must be -1 (choose), 0 (off), " + "or a thread count no greater than twice the available cores"); + if (c->vname == NULL && c->mname != NULL) c->vname = "base"; diff --git a/tests/test-zpool-mt.c b/tests/test-zpool-mt.c new file mode 100644 index 00000000..567b5bc9 --- /dev/null +++ b/tests/test-zpool-mt.c @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * With several threads writing one output, the pool guarantees + * containers reach the file in the order they were + * sealed. A thread's own payloads must therefore appear in the file in the + * order that thread wrote them. + * + * This is a per-thread claim, not a global one. Which thread wins a given + * container is a race, so nothing is asserted about how threads interleave -- + * only that no thread's payloads are reordered among themselves. + * + * Without a pool, container_submit() writes on the calling thread and two + * threads race for w_lock in send_buffer(), so containers can reach the file + * out of order. The check below therefore applies to the pooled runs only. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" + +#define NUM_THREADS 4 +#define PER_THREAD 5000 +#define BUFSZ NMSG_WBUFSZ_JUMBO + +static nmsg_msgmod_t mod; +static nmsg_output_t output; + +struct writer { + pthread_t thr; + unsigned id; +}; + +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-mt: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* On our way out regardless. */ + } + _exit(1); +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-mt: %s\n", what); + exit(1); +} + +static void * +writer_thread(void *arg) +{ + struct writer *w = (struct writer *) arg; + unsigned i; + + for (i = 0; i < PER_THREAD; i++) { + char payload[48]; + nmsg_message_t msg; + size_t len; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + len = snprintf(payload, sizeof(payload), "%u:%u", w->id, i); + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, + len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + } + + return (NULL); +} + +/* + * Read the file back and check that each thread's counters only ever increase. + * Returns the total number of payloads seen. + */ +static unsigned +verify_order(const char *path) +{ + unsigned last[NUM_THREADS]; + unsigned total = 0, i; + nmsg_input_t input; + nmsg_message_t msg; + int fd; + + for (i = 0; i < NUM_THREADS; i++) + last[i] = 0; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + while (nmsg_input_read(input, &msg) == nmsg_res_success) { + unsigned tid, seq; + void *data; + size_t len; + char buf[64]; + + if (nmsg_message_get_field(msg, "payload", 0, &data, + &len) != nmsg_res_success) + fail("nmsg_message_get_field() failed"); + + if (len >= sizeof(buf)) + fail("payload larger than expected"); + memcpy(buf, data, len); + buf[len] = '\0'; + + if (sscanf(buf, "%u:%u", &tid, &seq) != 2) + fail("payload did not parse"); + if (tid >= NUM_THREADS) + fail("payload carried an unknown thread id"); + + /* + * Counters start at 0, so 'last' holds the next value expected + * rather than the previous one seen. + */ + if (seq != last[tid]) { + fprintf(stderr, "test-zpool-mt: thread %u payload %u " + "arrived where %u was expected\n", + tid, seq, last[tid]); + exit(1); + } + last[tid] = seq + 1; + + nmsg_message_destroy(&msg); + total += 1; + } + + nmsg_input_close(&input); + close(fd); + + return (total); +} + +static void +run(const char *path, unsigned workers) +{ + struct writer writers[NUM_THREADS]; + unsigned i, total; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + nmsg_output_set_zlib_workers(output, workers); + + for (i = 0; i < NUM_THREADS; i++) { + writers[i].id = i; + if (pthread_create(&writers[i].thr, NULL, writer_thread, + &writers[i]) != 0) + fail("pthread_create() failed"); + } + + for (i = 0; i < NUM_THREADS; i++) + pthread_join(writers[i].thr, NULL); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + total = verify_order(path); + if (total != NUM_THREADS * PER_THREAD) { + fprintf(stderr, "test-zpool-mt: read %u payloads, expected %u\n", + total, NUM_THREADS * PER_THREAD); + exit(1); + } +} + +int main(void) { + char path[] = "/tmp/nmsg-zpool-mt.XXXXXX"; + int fd; + + signal(SIGALRM, on_alarm); + alarm(60); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + fd = mkstemp(path); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + + run(path, 1); + run(path, 4); + + unlink(path); + + return (0); +} \ No newline at end of file diff --git a/tests/test-zpool-order.c b/tests/test-zpool-order.c new file mode 100644 index 00000000..40d5ce0e --- /dev/null +++ b/tests/test-zpool-order.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The compressor pool must not change what gets written, only who compresses + * it. One producer writes the same payloads with a range of worker ceilings; + * every resulting file has to be byte for byte identical. + * + * A small bufsz is what makes this worth running: it puts many more containers + * in the file than a 1 MiB one would, so the slot ring wraps repeatedly and + * producers exercise the path where every worker is busy. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" + +#define NUM_PAYLOADS 4000 +#define BUFSZ NMSG_WBUFSZ_JUMBO + +/* Flush and check the file part-way through, at this payload. */ +#define CHECKPOINT_AT 1500 + +/* Enable the pool at this payload in the late-enable run. */ +#define LATE_ENABLE_AT 700 + +static nmsg_msgmod_t mod; + +/* + * A wedged pool would otherwise hang the test suite forever: automake has no + * per-test timeout, so the test has to impose its own. + */ +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-order: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* Nothing useful to do; we are on our way out regardless. */ + } + _exit(1); +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-order: %s\n", what); + exit(1); +} + +/* + * Payloads are deliberately short. A payload larger than bufsz would take the + * fragmenting path, whose fragment id is drawn from the RNG, and the output + * would then differ between two identical runs for reasons that have nothing + * to do with the pool. + */ +static nmsg_message_t +make_message(unsigned i) +{ + char payload[48]; + nmsg_message_t msg; + struct timespec ts; + size_t len; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + len = snprintf(payload, sizeof(payload), "payload %u", i); + if (len >= BUFSZ / 2) + fail("payload too large; would fragment"); + + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + /* Fixed, so two runs cannot differ on the timestamp. */ + ts.tv_sec = 1000000000 + i; + ts.tv_nsec = 0; + nmsg_message_set_time(msg, &ts); + + return (msg); +} + +/* Payloads readable from a path right now. */ +static unsigned +count_payloads(const char *path) +{ + nmsg_input_t input; + nmsg_message_t msg; + unsigned n = 0; + int fd; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + while (nmsg_input_read(input, &msg) == nmsg_res_success) { + nmsg_message_destroy(&msg); + n += 1; + } + + nmsg_input_close(&input); + close(fd); + + return (n); +} + +/* + * Write the corpus to 'path'. + * + * workers is the ceiling handed to nmsg_output_set_zlib_workers(). late_enable + * defers that call until the stream is already part-written, which is only safe + * because the pool seeds itself from the stream's ticket counter. rate throttles + * the writer so the committer falls behind and producers have to wait on a slot. + */ +static void +write_corpus(const char *path, unsigned workers, bool late_enable, nmsg_rate_t rate) +{ + nmsg_output_t output; + unsigned i; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + if (rate != NULL) + nmsg_output_set_rate(output, rate); + if (!late_enable) + nmsg_output_set_zlib_workers(output, workers); + + for (i = 0; i < NUM_PAYLOADS; i++) { + nmsg_message_t msg = make_message(i); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + + if (i == LATE_ENABLE_AT) { + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); + if (late_enable) + nmsg_output_set_zlib_workers(output, workers); + } + + /* + * A flush must leave everything written so far on disk, which + * is the contract every file rotation depends on. Checked while + * the output is still open, so it is the flush being tested and + * not the close. + */ + if (i == CHECKPOINT_AT) { + unsigned seen; + + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); + + seen = count_payloads(path); + if (seen != i + 1) { + fprintf(stderr, "test-zpool-order: flush left " + "%u of %u payloads on disk\n", seen, i + 1); + exit(1); + } + } + } + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +static void +compare(const char *ref, const char *path, const char *what) +{ + FILE *fa, *fb; + int ca, cb; + long off = 0; + + fa = fopen(ref, "rb"); + fb = fopen(path, "rb"); + if (fa == NULL || fb == NULL) + fail("fopen() for comparison failed"); + + do { + ca = getc(fa); + cb = getc(fb); + if (ca != cb) { + fprintf(stderr, "test-zpool-order: %s differs from the " + "inline output at byte %ld\n", what, off); + exit(1); + } + off += 1; + } while (ca != EOF); + + fclose(fa); + fclose(fb); +} + +int main(void) { + static const unsigned counts[] = { 1, 4, 8 }; + char ref[] = "/tmp/nmsg-zpool-ref.XXXXXX"; + char out[] = "/tmp/nmsg-zpool-out.XXXXXX"; + nmsg_rate_t rate; + unsigned i; + int fd; + + signal(SIGALRM, on_alarm); + alarm(30); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + /* mkstemp() only to get unique names; the writers reopen by path. */ + fd = mkstemp(ref); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + fd = mkstemp(out); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + + /* No pool: the reference every other run has to match. */ + write_corpus(ref, 0, false, NULL); + + if (count_payloads(ref) != NUM_PAYLOADS) + fail("reference output is missing payloads"); + + for (i = 0; i < sizeof(counts) / sizeof(counts[0]); i++) { + char what[64]; + + snprintf(what, sizeof(what), "output with %u worker(s)", counts[i]); + write_corpus(out, counts[i], false, NULL); + compare(ref, out, what); + } + + /* Pool enabled once the stream is already part-written. */ + write_corpus(out, 4, true, NULL); + compare(ref, out, "output with the pool enabled mid-stream"); + + /* + * Throttled, so the committer lags and the ring fills. This is the only + * run that reaches the slot wait in container_submit(). + */ + rate = nmsg_rate_init(200, 1); + if (rate == NULL) + fail("nmsg_rate_init() failed"); + write_corpus(out, 4, false, rate); + compare(ref, out, "rate-limited output"); + nmsg_rate_destroy(&rate); + + unlink(ref); + unlink(out); + + return (0); +} \ No newline at end of file From 5a3becee42f929fe96d8c79ca75bf14bcd8ab523 Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Fri, 21 Aug 2026 12:44:34 -0400 Subject: [PATCH 15/18] Size the reorder ring and worker ceiling per pool; rebuild on a ceiling change --- .gitignore | 6 + debian/libnmsg8.symbols | 3 +- doc/docbook/nmsgtool.docbook | 8 +- nmsg/output.c | 22 +- nmsg/output.h | 5 +- nmsg/output_async.c | 437 ++++++++++++++++++++--------------- nmsg/output_nmsg.c | 71 +++--- src/nmsgtool.c | 29 +-- src/nmsgtool.h | 16 +- src/process_args.c | 9 +- tests/test-zpool-order.c | 15 +- 11 files changed, 329 insertions(+), 292 deletions(-) diff --git a/.gitignore b/.gitignore index 5c94a3dd..e66417ad 100644 --- a/.gitignore +++ b/.gitignore @@ -65,5 +65,11 @@ tests/test-io tests/test-misc tests/test-parse tests/test-private +tests/test-zpool-mt +tests/test-zpool-order +tests/testzmq.json +tests/testzmq.sock tests/*/*.out tests/*/test.sh +.clang-format +.clangd diff --git a/debian/libnmsg8.symbols b/debian/libnmsg8.symbols index a4a7eb0c..dc09f02e 100644 --- a/debian/libnmsg8.symbols +++ b/debian/libnmsg8.symbols @@ -148,8 +148,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_set_operator@Base 0.5.0 nmsg_output_set_rate@Base 0.5.0 nmsg_output_set_source@Base 0.5.0 - nmsg_output_set_zlib_async@Base 0.5.0 - nmsg_output_set_zlib_workers@Base 0.5.0 + nmsg_output_set_zlib_workers@Base 1.4.0 nmsg_output_set_zlibout@Base 0.5.0 nmsg_output_write@Base 0.11.1 nmsg_pcap_filter@Base 0.6.5 diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index accdc4bc..e50004cb 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -670,9 +670,11 @@ n is a ceiling rather than a thread count: compressors are started only as load calls for them, so an output that never saturates never pays for - them. Values above twice the number of cores available to - the process are rejected, and a further internal limit - applies. + them. Compression is CPU-bound, so a thread per core is as + far as it can help: values above the number of cores + available to the process are rejected, and libnmsg applies + the same rule slightly tighter, since the buffer that feeds + the compressors is sized from their number. A value of chooses the ceiling: two per input, since a reader thread can saturate roughly diff --git a/nmsg/output.c b/nmsg/output.c index 36003960..d29680e1 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -424,9 +424,8 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { nmsg_res res; /* - * The type test comes first because 'stream' is a union member: on a - * pres or json output, reading stream->type would reinterpret the - * bytes of a different struct rather than fail. + * Type test first: 'stream' is a union member, so reading stream->type + * on a pres or json output would reinterpret another struct's bytes. */ if (output->type != nmsg_output_type_stream) return; @@ -434,9 +433,8 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { return; /* - * Unbuffered output flushes a container per message, so a pool would - * spend a ticket, a condvar signal and a wakeup per message to compress - * a single payload. + * Unbuffered flushes a container per message, so a pool would spend a + * ticket and a wakeup per message to compress a single payload. */ if (workers > 0 && !output->stream->buffered) { _nmsg_dprintf(1, "%s: ignored: not available on unbuffered output\n", @@ -445,9 +443,8 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { } /* - * Nothing to return an error through, so both paths are logged. Failing - * to start the pool is survivable, but disabling it after writing can - * strand an error a worker had recorded. + * Nothing to return an error through, so both paths log: disabling a + * pool after writing can strand an error a worker recorded. */ if (workers > 0) { res = _output_async_init(output, workers); @@ -601,7 +598,12 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { pthread_mutex_init(&output->stream->w_lock, NULL); pthread_cond_init(&output->stream->c_drained, NULL); - /* enable container sequencing */ + /* + * Enable container sequencing. Sock and zmq only, which is what lets + * the async compressor number containers in compression order; + * widening this test needs _output_nmsg_container_compress() looked + * at. + */ if (output->stream->type == nmsg_stream_type_sock || output->stream->type == nmsg_stream_type_zmq) { diff --git a/nmsg/output.h b/nmsg/output.h index eeec8a68..791203ec 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -392,8 +392,9 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); * the container itself, exactly as it would with no pool, so this is never * slower than leaving it off. * - * Writes stay in the order the containers were filled, so the output is - * byte-identical whatever \a workers is set to. + * Containers are written in the order they were filled, whatever \a workers is + * set to. With a single writing thread that makes the output byte-identical to + * compressing inline; with several, only the write order is guaranteed. * * \a workers is a ceiling rather than an allocation: threads are started as * load calls for them, so an output that never saturates never pays for them. diff --git a/nmsg/output_async.c b/nmsg/output_async.c index cb9dd77f..0d9b0d2e 100644 --- a/nmsg/output_async.c +++ b/nmsg/output_async.c @@ -15,55 +15,46 @@ */ /* - * The compressor pool for a stream output. - * - * Everything here is private to this unit: the ring, its slot states and the - * threads that walk it. output_nmsg.c hands a sealed container over with - * _output_async_submit() and gets the bytes written for it, and calls back - * into that file for the actual compressing and writing. + * The compressor pool for a stream output. The reorder buffer, its slot states + * and the threads that walk it are private here; output_nmsg.c submits sealed + * containers and supplies the compress and write callbacks. */ /* Import. */ #include "private.h" +#ifdef __linux__ +#include +#endif /* __linux__ */ + /* Data structures. */ /* - * A compressor pool: a ring of slots, up to nworkers compressor threads and - * one committer thread. - * - * A container is given a ticket when it is sealed, under c_lock, so tickets - * follow the order the containers were closed in. Slot i serves every ticket - * with (ticket % depth) == i, so the producer of ticket T waits only for - * ticket T - depth to have been written. - * - * Compression runs on whichever thread is free. Only the committer writes, and - * only in ticket order, so the byte stream is identical to the synchronous - * path however many workers are running. + * A compressor pool: a ticket reorder buffer, up to nworkers compressor + * threads and one committer thread. * - * A producer that finds every worker busy compresses the container itself - * rather than waiting. That is exactly what the synchronous path does, so the - * pool is never slower than no pool at all. + * Containers are ticketed under c_lock as they are sealed. Compression runs on + * whichever thread is free, but only the committer writes and only in ticket + * order, so the byte stream matches the synchronous path. * - * Workers are spawned on demand rather than up front, so 'nworkers' is a - * ceiling and not an allocation. Once spawned a worker lives until the pool is destroyed. + * A producer that finds every worker busy compresses inline rather than wait, + * so the pool is never slower than no pool. Workers spawn on demand, making + * 'nworkers' a ceiling rather than an allocation. */ /* - * Ring size, independent of the worker ceiling. A slot itself is tiny; what it - * costs is the container it points at while a ticket is in flight, so the depth - * bounds worst-case backlog rather than resident memory. The margin keeps slots - * available for producers to deposit into while every worker is busy, and - * capping the ceiling at depth - margin stops the pool having more compressors - * than places to put their output. + * Slots kept free for producers to deposit inline-compressed containers into + * while every worker is busy. Without it a saturated pool would have nowhere + * left to put anything. */ -#define ASYNC_RING_DEPTH 32 -#define ASYNC_RING_MARGIN 8 -#define ASYNC_MAX_WORKERS (ASYNC_RING_DEPTH - ASYNC_RING_MARGIN) +#define ASYNC_REORDER_MARGIN 8 + +/* Smallest reorder buffer worth allocating. */ +#define ASYNC_DEPTH_MIN 16 typedef enum { - slot_empty = 0, /* Free. */ + slot_empty = 0, /* Free. */ slot_work, /* Container waiting for a compressor. */ slot_taken, /* A worker or a producer is compressing it. */ slot_done, /* Compressed, waiting its turn to be written. */ @@ -71,35 +62,40 @@ typedef enum { } async_slot_state; struct async_slot { - async_slot_state state; - nmsg_container_t co; /* slot_work, slot_frag */ - uint8_t *buf; /* slot_done */ - size_t buf_len; - nmsg_res res; + async_slot_state state; + nmsg_container_t co; /* slot_work, slot_frag */ + uint8_t *buf; /* slot_done */ + size_t buf_len; + nmsg_res res; }; struct nmsg_ostr_async { - pthread_mutex_t lock; - pthread_cond_t work_ready; /* A slot became slot_work. */ - pthread_cond_t commit_ready; /* The committer's slot is ready. */ - pthread_cond_t slot_free; /* A slot became slot_empty. */ - struct async_slot *slots; - unsigned depth; - unsigned nworkers; /* Ceiling; workers start on demand. */ - unsigned nstarted; /* Workers that exist and must be joined. */ - unsigned busy; /* Containers assigned to workers. */ - uint64_t issued; /* Highest ticket claimed, plus one. */ - uint64_t commit_next; /* Ticket allowed to write now. */ - bool shutdown; - bool started; /* Committer exists; must be joined. */ - bool failed; /* No committer; stay inline. */ - bool spawn_failed; /* Worker spawn failed; logged once. */ - pthread_t *workers; - pthread_t committer; - nmsg_res first_error; /* Sticky; surfaced by flush. */ - nmsg_output_t output; - uint64_t n_inline; /* Containers a producer compressed. */ - uint64_t n_waited; /* Producers that waited for a slot. */ + pthread_mutex_t lock; + pthread_cond_t work_ready; /* A slot became slot_work. */ + pthread_cond_t commit_ready; /* The committer's slot is ready. */ + pthread_cond_t slot_free; /* A slot became slot_empty. */ + /* + * The ticket reorder buffer. Slot i serves every ticket with (ticket % + * depth) == i, so a producer runs at most 'depth' tickets ahead of + * commit_next and waits only for ticket T - depth to be written. + */ + struct async_slot *slots; + unsigned depth; + unsigned nworkers; /* Ceiling; workers start on demand. */ + unsigned nstarted; /* Workers that exist and must be joined. */ + unsigned busy; /* Containers assigned to workers. */ + uint64_t issued; /* Highest ticket claimed, plus one. */ + uint64_t commit_next; /* Ticket allowed to write now. */ + bool shutdown; + bool started; /* Committer exists; must be joined. */ + bool failed; /* No committer; stay inline. */ + bool spawn_failed; /* Worker spawn failed; logged once. */ + pthread_t *workers; + pthread_t committer; + nmsg_res first_error; /* Sticky; surfaced by flush. */ + nmsg_output_t output; + uint64_t n_inline; /* Containers a producer compressed. */ + uint64_t n_waited; /* Producers that waited for a slot. */ }; /* @@ -118,8 +114,7 @@ _output_async_ref(struct nmsg_stream_output *ostr) } /* Drop a reference taken by _output_async_ref() and wake any waiting teardown. */ -void -_output_async_unref(struct nmsg_stream_output *ostr) +void _output_async_unref(struct nmsg_stream_output *ostr) { pthread_mutex_lock(&ostr->c_lock); assert(ostr->so_inflight > 0); @@ -128,25 +123,94 @@ _output_async_unref(struct nmsg_stream_output *ostr) pthread_mutex_unlock(&ostr->c_lock); } +/* + * How many slots a pool of this size needs. + * + * One slot per worker covers everything that can be in flight, and the margin + * leaves room to deposit into. Past that, more depth only defers backpressure, + * and what it would be covering for is a slow write -- which is single-threaded + * on the committer, and no amount of depth helps. + * + * Sized here rather than fixed because the worker count spans an order of + * magnitude across the boxes this runs on, and a buffer sized for the largest + * is megabytes that a two-worker output never touches. + */ +static unsigned +async_depth_for(unsigned nworkers) +{ + unsigned depth = nworkers + ASYNC_REORDER_MARGIN; + + return (depth < ASYNC_DEPTH_MIN ? ASYNC_DEPTH_MIN : depth); +} + +/* + * Cores this process may run on. nmsgtool computes the same thing for its own + * sizing, but sees only the public header and cannot reach this one. + */ +static long +async_ncpu(void) +{ + long ncpu = -1; +#ifdef __linux__ + cpu_set_t set; + + if (sched_getaffinity(0, sizeof(set), &set) == 0) + ncpu = CPU_COUNT(&set); +#endif /* __linux__ */ + + if (ncpu < 1) + ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if (ncpu < 1) + ncpu = 1; + + return (ncpu); +} + +/* + * Compressor threads one output may have. Compression is CPU-bound, so more + * than one thread per core cannot help; past that they only add context + * switches and reorder buffer. + * + * This is a backstop for callers passing an arbitrary count. nmsgtool sizes + * its own request against the readers it is actually running. + */ +static unsigned +async_max_workers(void) +{ + long ncpu = async_ncpu(); + + if (ncpu < ASYNC_DEPTH_MIN) + ncpu = ASYNC_DEPTH_MIN; + + return ((unsigned)(ncpu - ASYNC_REORDER_MARGIN)); +} + nmsg_res -_output_async_init(nmsg_output_t output, unsigned nworkers) { +_output_async_init(nmsg_output_t output, unsigned nworkers) +{ struct nmsg_stream_output *ostr = output->stream; - struct nmsg_ostr_async *pool; - nmsg_res res, old_res = nmsg_res_success; + struct nmsg_ostr_async *pool; + nmsg_res res, old_res = nmsg_res_success; + unsigned depth, max_workers; + bool same_ceiling; if (nworkers == 0) return (nmsg_res_success); - if (nworkers > ASYNC_MAX_WORKERS) - nworkers = ASYNC_MAX_WORKERS; + max_workers = async_max_workers(); + if (nworkers > max_workers) + nworkers = max_workers; + + depth = async_depth_for(nworkers); /* - * An existing pool's ceiling cannot be changed in place, since its - * worker array and ring are already sized, so a different count means - * building a replacement. Do that before tearing the old one down: if - * the allocation fails there is still a working pool to keep. + * A pool's ceiling cannot change in place, so a different count means + * a replacement. Build it before tearing the old one down, so a failed + * allocation leaves the working pool in place. */ - if (ostr->so_pool != NULL && ostr->so_pool->nworkers == nworkers) + same_ceiling = ostr->so_pool != NULL && + ostr->so_pool->nworkers == nworkers; + if (same_ceiling) return (nmsg_res_success); pool = calloc(1, sizeof(*pool)); @@ -155,7 +219,7 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) { res = nmsg_res_memfail; - pool->slots = calloc(ASYNC_RING_DEPTH, sizeof(*pool->slots)); + pool->slots = calloc(depth, sizeof(*pool->slots)); if (pool->slots == NULL) goto fail_slots; @@ -174,7 +238,7 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) { if (pthread_cond_init(&pool->slot_free, NULL) != 0) goto fail_slot_free; - pool->depth = ASYNC_RING_DEPTH; + pool->depth = depth; pool->nworkers = nworkers; pool->output = output; @@ -184,19 +248,15 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) { pthread_mutex_lock(&ostr->c_lock); /* - * Tickets count for the life of the stream, not the life of the pool, - * so a pool built after the first write must start where the stream has - * got to. Seeded under c_lock, and the pool is published in the same - * hold, so a ticket either predates the pool and is compressed inline or - * belongs to it and is at or above commit_next -- never below, where the - * committer would wait for it forever. + * Tickets span the stream, not the pool, so a pool built mid-stream + * must start where the stream got to. Seeded and published in one + * c_lock hold, so a ticket either predates the pool or is at or above + * commit_next -- never below, where the committer would wait for it + * forever. */ pool->commit_next = pool->issued = ostr->so_ticket; - /* - * Carry any error the previous pool recorded but had not yet reported, - * so replacing a pool does not swallow a failed write. - */ + /* Carry the old pool's unreported error rather than swallow it. */ pool->first_error = old_res; ostr->so_pool = pool; @@ -222,26 +282,25 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) { } /* - * Stop the pool and reclaim it. Everything still queued is written first. - * Must run before the stream's fd, random and locks go away, since the threads - * use all of them. + * Stop the pool and reclaim it, writing everything still queued. Must run + * before the stream's fd, random and locks go away; the threads use all three. */ nmsg_res -_output_async_destroy(nmsg_output_t output) { +_output_async_destroy(nmsg_output_t output) +{ struct nmsg_stream_output *ostr = output->stream; - struct nmsg_ostr_async *pool = ostr->so_pool; - nmsg_res res; - bool started; - unsigned i, nstarted; + struct nmsg_ostr_async *pool = ostr->so_pool; + nmsg_res res; + bool started; + unsigned i, nstarted; if (pool == NULL) return (nmsg_res_success); /* - * Stop handing tickets to the pool, then wait for the ones already - * handed out to arrive. Both happen under c_lock, which is what orders - * them against ticket issuance: once this returns, no producer is still - * on its way here holding a ticket the committer will wait for. + * Stop issuing tickets to the pool, then wait for the outstanding ones + * to arrive. Both under c_lock, which orders them against issuance: + * once this returns, no producer is still en route with a ticket. */ pthread_mutex_lock(&ostr->c_lock); ostr->so_pool_closing = true; @@ -250,27 +309,22 @@ _output_async_destroy(nmsg_output_t output) { pthread_mutex_unlock(&ostr->c_lock); pthread_mutex_lock(&pool->lock); - pool->shutdown = true; /* Set under the lock: a thread about */ - started = pool->started; /* to wait would miss the wakeup. */ + pool->shutdown = true; /* Set under the lock: a thread about */ + started = pool->started; /* to wait would miss the wakeup. */ nstarted = pool->nstarted; pthread_cond_broadcast(&pool->work_ready); pthread_cond_broadcast(&pool->commit_ready); pthread_cond_broadcast(&pool->slot_free); pthread_mutex_unlock(&pool->lock); - /* - * Workers and committer are joined on their own counters. - */ for (i = 0; i < nstarted; i++) pthread_join(pool->workers[i], NULL); if (started) pthread_join(pool->committer, NULL); if (pool->n_inline > 0 || pool->n_waited > 0) - _nmsg_dprintf(2, "%s: %u of %u worker(s) started; %" PRIu64 - " container(s) compressed by the reader, %" PRIu64 - " wait(s) for a free slot\n", __func__, nstarted, - pool->nworkers, pool->n_inline, pool->n_waited); + _nmsg_dprintf(2, "%s: %u of %u worker(s) started, %u slot(s); %" PRIu64 " container(s) compressed by the reader, %" PRIu64 " wait(s) for a free slot\n", __func__, nstarted, + pool->nworkers, pool->depth, pool->n_inline, pool->n_waited); /* Read after the joins; the threads write it until they exit. */ res = pool->first_error; @@ -300,24 +354,42 @@ async_record_error(struct nmsg_ostr_async *pool, nmsg_res res) } /* - * Compressor thread. Takes any slot that needs compressing, in whatever order - * they become ready: compression order does not matter, only write order does, - * and the committer enforces that. + * The slot this ticket owns is still held by the ticket 'depth' earlier. + * Caller holds pool->lock. + */ +static bool +async_slot_busy(const struct nmsg_ostr_async *pool, uint64_t ticket) +{ + return (ticket >= pool->commit_next + pool->depth); +} + +/* + * Every ticket the pool was given has been written. Caller holds pool->lock. + */ +static bool +async_all_written(const struct nmsg_ostr_async *pool) +{ + return (pool->commit_next >= pool->issued); +} + +/* + * Compressor thread. Takes any slot needing work, in any order: only write + * order matters, and the committer enforces that. */ static void * async_worker(void *arg) { - struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *)arg; pthread_mutex_lock(&pool->lock); for (;;) { struct async_slot *slot = NULL; - nmsg_container_t co; - uint8_t *buf; - size_t buf_len; - nmsg_res res; - unsigned i; + nmsg_container_t co; + uint8_t *buf; + size_t buf_len; + nmsg_res res; + unsigned i; for (i = 0; i < pool->depth; i++) { if (pool->slots[i].state == slot_work) { @@ -328,10 +400,9 @@ async_worker(void *arg) if (slot == NULL) { /* - * Nothing to compress. Exit only once the producers - * have stopped, so a container queued just before - * shutdown is still compressed and written; that - * happens on every clean SIGTERM. + * Exit only once producers have stopped, so a + * container queued just before shutdown is still + * written. Happens on every clean SIGTERM. */ if (pool->shutdown) break; @@ -347,7 +418,7 @@ async_worker(void *arg) res = _output_nmsg_container_compress(pool->output, &co, &buf, &buf_len); pthread_mutex_lock(&pool->lock); - pool->busy--; /* Counted at deposit; see container_submit(). */ + pool->busy--; /* Counted at deposit; see container_submit(). */ slot->buf = buf; slot->buf_len = buf_len; slot->res = res; @@ -361,25 +432,23 @@ async_worker(void *arg) } /* - * The only thread that writes. It takes tickets strictly in order, so the file - * is byte-identical to what the synchronous path would have produced no matter - * how many workers compressed in parallel. - * - * It is also the only caller of _output_nmsg_frag_write(). + * The only thread that writes, and the only caller of + * _output_nmsg_frag_write(). Takes tickets strictly in order, so the file + * matches what the synchronous path would have produced. */ static void * async_committer(void *arg) { - struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *) arg; + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *)arg; pthread_mutex_lock(&pool->lock); for (;;) { struct async_slot *slot = &pool->slots[pool->commit_next % pool->depth]; - uint8_t *buf; - size_t buf_len; - nmsg_container_t co; - nmsg_res res = nmsg_res_success; + uint8_t *buf; + size_t buf_len; + nmsg_container_t co; + nmsg_res res = nmsg_res_success; switch (slot->state) { case slot_done: @@ -390,9 +459,8 @@ async_committer(void *arg) pthread_mutex_unlock(&pool->lock); /* - * _output_nmsg_send_buffer() frees buf. It is not called when the - * compression failed, but then buf is NULL anyway; see - * _output_nmsg_container_compress(). + * _output_nmsg_send_buffer() frees buf; not called on + * a compression failure, where buf is NULL anyway. */ if (res == nmsg_res_success) res = _output_nmsg_send_buffer(pool->output, buf, buf_len); @@ -405,7 +473,7 @@ async_committer(void *arg) slot->co = NULL; pthread_mutex_unlock(&pool->lock); - /* _output_nmsg_frag_write() takes the container by value and destroys it. */ + /* Takes the container by value and destroys it. */ res = _output_nmsg_frag_write(pool->output, co); pthread_mutex_lock(&pool->lock); @@ -415,11 +483,10 @@ async_committer(void *arg) case slot_work: case slot_taken: /* - * The next ticket is not ready. Exit only when the - * producers have stopped and every ticket they issued - * has been written. + * Not ready. Exit only once producers have stopped and + * every ticket they issued has been written. */ - if (pool->shutdown && pool->commit_next >= pool->issued) + if (pool->shutdown && async_all_written(pool)) goto out; pthread_cond_wait(&pool->commit_ready, &pool->lock); continue; @@ -438,14 +505,10 @@ async_committer(void *arg) } /* - * Start the committer. Called under pool->lock on first submit rather than when - * the output is configured, because nmsgtool creates its outputs before it - * daemonizes, and daemonize() is a bare fork() which no thread survives. - * Starting on first write puts the threads in whichever process does the - * writing. - * - * Only the committer starts here. Workers are added by async_spawn_worker() as - * load calls for them. + * Start the committer, under pool->lock on first submit rather than at + * configure time: nmsgtool creates its outputs before daemonize(), which is a + * bare fork() no thread survives. Workers are added later by + * async_spawn_worker(). */ static void async_start(struct nmsg_ostr_async *pool) @@ -455,10 +518,9 @@ async_start(struct nmsg_ostr_async *pool) pthread_res = pthread_create(&pool->committer, NULL, async_committer, pool); if (pthread_res != 0) { /* - * Nothing can be written without a committer, so give up on the - * pool entirely and compress inline from here on. No worker has - * been created yet and no container has been deposited, so there - * is nothing to unwind. + * Nothing can be written without a committer, so abandon the + * pool and compress inline. Nothing has been deposited yet to + * unwind. */ pool->failed = true; _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, @@ -471,12 +533,10 @@ async_start(struct nmsg_ostr_async *pool) /* * Add a compressor thread, up to the ceiling. Called under pool->lock when a - * producer finds every existing worker busy, so the pool grows to the load it - * actually sees instead of to the configured ceiling. + * producer finds every worker busy, so the pool grows to the load it sees. * - * Returns false if the thread could not be created, which is not fatal: the - * caller compresses that container itself and the pool keeps running with the - * workers it has. + * Returns false if the thread could not be created: not fatal, the caller + * compresses that container itself. */ static bool async_spawn_worker(struct nmsg_ostr_async *pool) @@ -503,10 +563,9 @@ async_spawn_worker(struct nmsg_ostr_async *pool) /* * Wait until every ticket issued so far has been written, and take any error * the pool recorded. Under nmsg_io this cannot starve: check_close_event() - * holds io_output->refcount across the write and call_close_fp() waits for it - * to drop, so no other thread is inside nmsg_output_write() while a close runs. - * A caller driving nmsg_output_flush() directly from several threads has no - * such guarantee. + * holds io_output->refcount across the write, so no writer is inside + * nmsg_output_write() while a close runs. Callers driving nmsg_output_flush() + * from several threads have no such guarantee. */ nmsg_res _output_async_drain(struct nmsg_ostr_async *pool) @@ -517,7 +576,7 @@ _output_async_drain(struct nmsg_ostr_async *pool) return (nmsg_res_success); pthread_mutex_lock(&pool->lock); - while (!pool->failed && pool->commit_next < pool->issued) + while (!pool->failed && !async_all_written(pool)) pthread_cond_wait(&pool->slot_free, &pool->lock); res = pool->first_error; pool->first_error = nmsg_res_success; @@ -528,35 +587,37 @@ _output_async_drain(struct nmsg_ostr_async *pool) /* * Hand a sealed container to the pool, consuming it only if the pool takes it. - * Returns false if it does not, and the caller writes the container itself. - * The pool reference is released either way. + * Returns false if it does not, leaving it for the caller to write. The pool + * reference is released either way. * - * On success *res_out carries the error from an EARLIER container, since the - * one just handed over has not been written yet. + * *res_out carries an EARLIER container's error; this one is not written yet. */ -bool -_output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, - nmsg_container_t *co, bool is_frag, uint64_t ticket, - nmsg_res *res_out) +bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, + nmsg_container_t *co, bool is_frag, uint64_t ticket, + nmsg_res *res_out) { struct nmsg_stream_output *ostr = output->stream; - struct async_slot *slot; - nmsg_res res = nmsg_res_success; - uint8_t *buf; - size_t buf_len; - bool inline_compress = false; + struct async_slot *slot; + nmsg_res res = nmsg_res_success; + uint8_t *buf; + size_t buf_len; + bool inline_compress = false; + bool committer_needed, pool_usable; + bool worker_idle, below_ceiling; pthread_mutex_lock(&pool->lock); - if (!pool->started && !pool->failed && !pool->shutdown) + committer_needed = !pool->started && !pool->failed && !pool->shutdown; + if (committer_needed) async_start(pool); /* - * Only reachable when the committer could not be started: teardown drains - * outstanding tickets before setting shutdown, so a ticket that got this - * far still has a pool to go to. + * Read after the attempt, since async_start() sets started or failed. + * Only false when the committer could not start: teardown drains + * outstanding tickets before setting shutdown. */ - if (!pool->started || pool->failed || pool->shutdown) { + pool_usable = pool->started && !pool->failed && !pool->shutdown; + if (!pool_usable) { pthread_mutex_unlock(&pool->lock); _output_async_unref(ostr); return (false); @@ -565,13 +626,12 @@ _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, slot = &pool->slots[ticket % pool->depth]; /* - * Wait for the slot this ticket owns, which the ticket 'depth' earlier - * releases when it is written. Only reached when the writer has fallen - * a whole ring behind. + * Wait for the slot this ticket owns, released by the ticket 'depth' + * earlier. Only reached when the writer is a full 'depth' behind. */ - if (ticket >= pool->commit_next + pool->depth) { + if (async_slot_busy(pool, ticket)) { pool->n_waited++; - while (ticket >= pool->commit_next + pool->depth && !pool->shutdown) + while (async_slot_busy(pool, ticket) && !pool->shutdown) pthread_cond_wait(&pool->slot_free, &pool->lock); } @@ -580,29 +640,24 @@ _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, if (ticket >= pool->issued) pool->issued = ticket + 1; + worker_idle = pool->busy < pool->nstarted; + below_ceiling = pool->nstarted < pool->nworkers; + if (is_frag) { /* Only the committer fragments; see async_committer(). */ slot->co = *co; *co = NULL; slot->state = slot_frag; pthread_cond_broadcast(&pool->commit_ready); - } else if (pool->busy < pool->nstarted || - (pool->nstarted < pool->nworkers && async_spawn_worker(pool))) - { - /* - * A worker is free, or the ceiling left room to add one: hand - * the container over and get back to reading. - */ + } else if (worker_idle || (below_ceiling && async_spawn_worker(pool))) { + /* Hand the container over and get back to reading. */ slot->co = *co; *co = NULL; slot->state = slot_work; pool->busy++; pthread_cond_signal(&pool->work_ready); } else { - /* - * Every worker is busy and the ceiling is reached. Compress it - * here rather than wait, then deposit the result and return. - */ + /* Everyone busy and at the ceiling: compress here. */ slot->state = slot_taken; pool->n_inline++; inline_compress = true; diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index cb8b2065..dfb80d3a 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -24,13 +24,8 @@ static nmsg_res container_write(nmsg_output_t, nmsg_container_t*); static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool, uint64_t, struct nmsg_ostr_async *); -/* Data structures. */ - - /* Internal functions. */ - - nmsg_res _output_nmsg_flush(nmsg_output_t output) { struct nmsg_stream_output *ostr = output->stream; @@ -63,9 +58,8 @@ _output_nmsg_flush(nmsg_output_t output) { pthread_mutex_unlock(&ostr->c_lock); /* - * Submitted outside c_lock. Flush runs on every file rotation, and a - * submit can wait for a free slot; doing that under c_lock would hold - * off every reader thread for the length of a compression. + * Submitted outside c_lock: a submit can wait for a free slot, and + * holding c_lock across that would stall every reader thread. */ if (old_c != NULL) { nmsg_res sub_res; @@ -75,10 +69,7 @@ _output_nmsg_flush(nmsg_output_t output) { res = sub_res; } - /* - * A flush means the data has been written, so wait out anything the - * pool is still holding. - */ + /* A flush means written, so wait out anything the pool still holds. */ drain_res = _output_async_drain(pool); if (res == nmsg_res_success) res = drain_res; @@ -139,11 +130,11 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { * container for the other threads to use. */ is_buffered = ostr->buffered; /* Save this value. */ - if ((res == nmsg_res_container_full) || - (res == nmsg_res_success && is_buffered == false) || - (res == nmsg_res_container_overfull)) { - must_flush = true; /* Will flush container below. */ + must_flush = (res == nmsg_res_container_full) || + (res == nmsg_res_success && is_buffered == false) || + (res == nmsg_res_container_overfull); + if (must_flush) { /* Create replacement container. */ new_c = nmsg_container_init(ostr->bufsz); if (new_c == NULL) { @@ -157,12 +148,10 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { ostr->c = new_c; /* - * Ticket taken here, under c_lock, because this is where the - * container's contents become final. Taking it in - * container_submit() would order the containers by which - * thread won the race after the unlock, which is not the - * order they were filled in. The pool is claimed in the same - * hold, so the ticket and the pool that will serve it are + * Ticket taken under c_lock, where the container's contents + * become final. Taking it after the unlock would order + * containers by which thread won the race, not by fill order. + * The pool is claimed in the same hold, so ticket and pool are * chosen together. */ ticket = ostr->so_ticket++; @@ -199,7 +188,7 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { */ nmsg_res _output_nmsg_container_compress(nmsg_output_t output, nmsg_container_t *co, - uint8_t **buf, size_t *buf_len) + uint8_t **buf, size_t *buf_len) { struct nmsg_stream_output *ostr = output->stream; nmsg_res res; @@ -207,8 +196,10 @@ _output_nmsg_container_compress(nmsg_output_t output, nmsg_container_t *co, uint8_t *shrunk; /* - * Multiple threads can enter here at once, so the numbers are handed out - * in compression order rather than write order. + * Multiple threads can enter here at once, so numbers go out in + * compression order, not write order. Safe only because pools are + * file-only and file outputs leave do_sequence false; a pool on a + * sequenced output would scramble them. */ seq = atomic_fetch_add_explicit(&ostr->so_sequence_num, 1, memory_order_relaxed); @@ -223,9 +214,9 @@ _output_nmsg_container_compress(nmsg_output_t output, nmsg_container_t *co, } /* - * nmsg_container_serialize() returns the base of an allocation sized - * for the worst case, twice the unpacked estimate. A slot holds that - * until every earlier ticket has been written, so hand the rest back. + * serialize() allocates for the worst case, twice the unpacked + * estimate. A slot pins that until every earlier ticket is written, so + * hand it back. */ shrunk = realloc(*buf, *buf_len); if (shrunk != NULL) @@ -252,12 +243,6 @@ container_write(nmsg_output_t output, nmsg_container_t *co) return (_output_nmsg_send_buffer(output, buf, buf_len)); } - - - - - - /* Compress and write on the calling thread. The container is consumed. */ static nmsg_res container_submit_inline(nmsg_output_t output, nmsg_container_t *co, bool is_frag) @@ -265,10 +250,7 @@ container_submit_inline(nmsg_output_t output, nmsg_container_t *co, bool is_frag if (is_frag) { nmsg_container_t tmp = *co; - /* - * _output_nmsg_frag_write() takes the container by value and destroys it - * internally, unlike container_write(). - */ + /* Takes the container by value and destroys it. */ *co = NULL; return (_output_nmsg_frag_write(output, tmp)); } @@ -278,19 +260,20 @@ container_submit_inline(nmsg_output_t output, nmsg_container_t *co, bool is_frag /* * Hand a finished container to the pool, or process it inline if there is no - * pool or the pool declines it. The container is consumed either way. - * - * 'pool' is the reference taken under c_lock when the ticket was issued, so it - * is NULL exactly when the ticket was never promised to a pool. + * pool or it declines. The container is consumed either way. 'pool' is the + * reference taken when the ticket was issued, so it is NULL exactly when the + * ticket was never promised to a pool. */ static nmsg_res container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag, uint64_t ticket, struct nmsg_ostr_async *pool) { nmsg_res res; + bool taken_by_pool; - if (pool != NULL && - _output_async_submit(pool, output, co, is_frag, ticket, &res)) + taken_by_pool = pool != NULL && + _output_async_submit(pool, output, co, is_frag, ticket, &res); + if (taken_by_pool) return (res); return (container_submit_inline(output, co, is_frag)); diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 6b93db2d..d1a65327 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -463,27 +463,13 @@ nmsgtool_ncpu(void) { } /* - * How many compressor threads an output should get. + * How many compressor threads an output should get. A negative --zasync means + * auto: demand is about two workers per input, since a reader can saturate + * roughly one core compressing, and supply is the cores the readers leave. * - * A negative --zasync means auto. Two things bound the answer: - * - * Demand scales with the inputs. Each reader thread can saturate roughly one - * core compressing, and covering that takes about two workers per input. - * - * Supply is the cores the readers leave. Workers beyond that only contend. - * - * The floor matters because one input can carry several cores' worth on its - * own, so a count that merely followed the input count would under-serve - * exactly the case the pool exists for. - * - * The budget is then split across the file outputs, since each gets its own - * pool and they share these same cores. Each output keeps at least one worker: - * one is still enough to take compression off the reader, which is the point. - * - * Getting it wrong is cheap in both directions. Too small a pool degrades to - * the behaviour of no pool rather than stalling, because a reader that finds - * every worker busy compresses the container itself. Too large only raises a - * ceiling that is never reached, since workers are started on demand. + * The budget is split across the file outputs, which each get their own pool + * and share these cores. One worker is still enough to take compression off + * the reader, so no output drops below that. */ static unsigned zworkers_count(nmsgtool_ctx *c) { @@ -515,8 +501,7 @@ zworkers_count(nmsgtool_ctx *c) { /* * Resolve --zasync and apply it to the outputs that already exist. Deferred to * the end of process_args() because the input count is not final until then: a - * channel alias (-C) expands to its sockets after the outputs have been - * created, which is exactly the case the pool is sized for. + * channel alias (-C) expands to its sockets after the outputs are created. */ void setup_nmsg_output_workers(nmsgtool_ctx *c) { diff --git a/src/nmsgtool.h b/src/nmsgtool.h index ca590fd3..baad358e 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -52,19 +52,19 @@ VECTOR_GENERATE(statsmod_vec, nmsg_statsmod_t) VECTOR_GENERATE(output_vec, nmsg_output_t) /* - * Floor for an automatically chosen compressor pool. A single input socket can - * carry well over one core's worth of compression on its own -- at 120 MB/s it - * needed four workers, where two still lost 13 % -- so the count cannot simply - * follow the socket count downwards. + * Floor for an automatically chosen pool. One input socket can carry well over + * a core's worth of compression: at 120 MB/s it needed four workers, and two + * still lost 13 %. */ #define NMSGTOOL_ZWORKERS_MIN 4 /* - * Ceiling for an explicit --zasync. Workers start on demand, so this only - * bounds how far a saturated output may grow; libnmsg applies its own limit - * on top, since it cannot assume its caller validated anything. + * Ceiling for an explicit --zasync. Compression is CPU-bound, so a thread per + * core is as far as it can help; workers start on demand, so this only bounds + * how far a saturated output may grow. libnmsg applies the same rule a margin + * tighter, since it sizes the reorder buffer that serves them. */ -#define NMSGTOOL_ZWORKERS_MAX(ncpu) (2 * (ncpu)) +#define NMSGTOOL_ZWORKERS_MAX(ncpu) (ncpu) typedef struct { /* parameters */ diff --git a/src/process_args.c b/src/process_args.c index 6f94c203..2150cde6 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -127,7 +127,7 @@ process_args(nmsgtool_ctx *c) { if (c->zasync < -1 || c->zasync > NMSGTOOL_ZWORKERS_MAX(nmsgtool_ncpu())) usage("--zasync must be -1 (choose), 0 (off), " - "or a thread count no greater than twice the available cores"); + "or a thread count no greater than the available cores"); if (c->vname == NULL && c->mname != NULL) c->vname = "base"; @@ -414,10 +414,9 @@ process_args(nmsgtool_ctx *c) { } /* - * Size the compressor pool now that every input exists. Must follow - * the implicit output above, and must precede daemonize(): the pool's - * threads are started on first write, which happens in whichever - * process ends up doing the writing. + * Size the pool now that every input exists. Must follow the implicit + * output above and precede daemonize(): pool threads start on first + * write, in whichever process does the writing. */ setup_nmsg_output_workers(c); diff --git a/tests/test-zpool-order.c b/tests/test-zpool-order.c index 40d5ce0e..6b21de44 100644 --- a/tests/test-zpool-order.c +++ b/tests/test-zpool-order.c @@ -20,8 +20,9 @@ * every resulting file has to be byte for byte identical. * * A small bufsz is what makes this worth running: it puts many more containers - * in the file than a 1 MiB one would, so the slot ring wraps repeatedly and - * producers exercise the path where every worker is busy. + * in the file than a 1 MiB one would, so the reorder buffer wraps repeatedly + * and producers exercise the path where every worker is busy. The worker counts + * below span both buffer sizes, since the depth is derived from them. */ #include @@ -226,7 +227,11 @@ compare(const char *ref, const char *path, const char *what) } int main(void) { - static const unsigned counts[] = { 1, 4, 8 }; + /* + * 16 asks for a bigger reorder buffer than the rest; libnmsg clamps it + * to what the machine allows, which exercises the clamp either way. + */ + static const unsigned counts[] = { 1, 4, 8, 16 }; char ref[] = "/tmp/nmsg-zpool-ref.XXXXXX"; char out[] = "/tmp/nmsg-zpool-out.XXXXXX"; nmsg_rate_t rate; @@ -272,8 +277,8 @@ int main(void) { compare(ref, out, "output with the pool enabled mid-stream"); /* - * Throttled, so the committer lags and the ring fills. This is the only - * run that reaches the slot wait in container_submit(). + * Throttled, so the committer lags and the reorder buffer fills. This is + * the only run that reaches the slot wait in _output_async_submit(). */ rate = nmsg_rate_init(200, 1); if (rate == NULL) From aee2c8a94e3e01685644bf72abdf5e3dee736a69 Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Fri, 21 Aug 2026 13:44:38 -0400 Subject: [PATCH 16/18] Cull compressors idle past --zcull, never below --zmin; take the MRU worker --- .gitignore | 1 + Makefile.am | 10 + configure.ac | 3 + debian/libnmsg8.symbols | 1 + doc/docbook/nmsgtool.docbook | 47 +++- nmsg/output.c | 38 ++++ nmsg/output.h | 53 ++--- nmsg/output_async.c | 420 +++++++++++++++++++++++++++++------ nmsg/private.h | 13 ++ src/nmsgtool.c | 38 +++- src/nmsgtool.h | 9 + src/process_args.c | 12 + tests/test-zpool-cull.c | 329 +++++++++++++++++++++++++++ 13 files changed, 867 insertions(+), 107 deletions(-) create mode 100644 tests/test-zpool-cull.c diff --git a/.gitignore b/.gitignore index e66417ad..158f87fe 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ tests/test-io tests/test-misc tests/test-parse tests/test-private +tests/test-zpool-cull tests/test-zpool-mt tests/test-zpool-order tests/testzmq.json diff --git a/Makefile.am b/Makefile.am index f5aa64ee..f459a305 100644 --- a/Makefile.am +++ b/Makefile.am @@ -572,6 +572,16 @@ check_PROGRAMS += tests/test-zpool-mt tests_test_zpool_mt_SOURCES = tests/test-zpool-mt.c tests_test_zpool_mt_LDADD = nmsg/libnmsg.la +TESTS += tests/test-zpool-cull +check_PROGRAMS += tests/test-zpool-cull +tests_test_zpool_cull_LDFLAGS = -rdynamic +tests_test_zpool_cull_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS) +tests_test_zpool_cull_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) +tests_test_zpool_cull_SOURCES = tests/test-zpool-cull.c + DISTCLEANFILES += tests/group-operator-source-tests/test*.out DISTCLEANFILES += tests/nmsg-dns-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out diff --git a/configure.ac b/configure.ac index 1573e387..8f2cca9f 100644 --- a/configure.ac +++ b/configure.ac @@ -96,6 +96,9 @@ AC_CHECK_FUNCS([clock_gettime]) AC_SEARCH_LIBS([clock_nanosleep], [rt]) AC_CHECK_FUNCS([clock_nanosleep]) +AC_SEARCH_LIBS([pthread_condattr_setclock], [pthread]) +AC_CHECK_FUNCS([pthread_condattr_setclock]) + AC_SEARCH_LIBS([dlopen], [dl]) AC_CHECK_FUNCS([dlopen]) diff --git a/debian/libnmsg8.symbols b/debian/libnmsg8.symbols index dc09f02e..1a9515fd 100644 --- a/debian/libnmsg8.symbols +++ b/debian/libnmsg8.symbols @@ -148,6 +148,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_set_operator@Base 0.5.0 nmsg_output_set_rate@Base 0.5.0 nmsg_output_set_source@Base 0.5.0 + nmsg_output_set_zlib_cull@Base 1.4.0 nmsg_output_set_zlib_workers@Base 1.4.0 nmsg_output_set_zlibout@Base 0.5.0 nmsg_output_write@Base 0.11.1 diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index e50004cb..75dfa34e 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -669,8 +669,9 @@ n is a ceiling rather than a thread count: compressors are started only as load calls - for them, so an output that never saturates never pays for - them. Compression is CPU-bound, so a thread per core is as + for them, and given back once they go idle, so an output + that never saturates never pays for them. See + and . Compression is CPU-bound, so a thread per core is as far as it can help: values above the number of cores available to the process are rejected, and libnmsg applies the same rule slightly tighter, since the buffer that feeds @@ -707,6 +708,48 @@ + + secs + + Give up a compressor thread that has been idle for + secs seconds, 300 by default. A + value of never gives one up, leaving a + pool at the largest it ever needed to be. + + Containers go to the compressor used most recently, so + a pool that grew for a burst keeps the front of that order + busy and lets the rest fall quiet. The pool grows again on + demand, exactly as it did the first time, and a compressor + only ever leaves when it is holding nothing, so the output + is unaffected either way. + + This matters mainly for a long-running output that is + not being rotated: an output closed by + or gives its whole pool back at every + close regardless. + + + + + n + + Never let take an output + below n compressor threads, 1 by + default. A value of lets the pool empty + completely. + + This is a floor on what culling may take away, not a + number of threads to start: compressors are still started + only on demand, so an output that has never been busy is + running none of them whatever this is set to. A value above + the ceiling in force is lowered to it, which is reported at + . + + One thread per file output is not culled in any case: + the thread that does the writing is not a compressor. + + + diff --git a/nmsg/output.c b/nmsg/output.c index d29680e1..0e062fca 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -459,6 +459,42 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { } } +void +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, unsigned idle_secs) { + struct nmsg_stream_output *ostr; + struct nmsg_ostr_async *pool; + + /* Type test first, for the reason nmsg_output_set_zlib_workers() gives. */ + if (output->type != nmsg_output_type_stream) + return; + if (output->stream->type != nmsg_stream_type_file) + return; + + ostr = output->stream; + + /* + * Kept on the stream, not just in the pool: a ceiling change replaces + * the pool, and this way the two setters may be called in any order. + * Nothing is logged when there is no pool -- nmsgtool sets a policy on + * every output, so it would fire on a plain '--unbuffered -w'. + */ + pthread_mutex_lock(&ostr->c_lock); + ostr->so_zmin = min_workers; + ostr->so_zcull = idle_secs; + + /* + * Taken under c_lock so a teardown cannot free the pool underneath. + * Lock order is c_lock then pool->lock; nothing takes them the other way. + */ + pool = _output_async_ref(ostr); + pthread_mutex_unlock(&ostr->c_lock); + + if (pool != NULL) { + _output_async_set_cull(pool, min_workers, idle_secs); + _output_async_unref(ostr); + } +} + void nmsg_output_set_endline(nmsg_output_t output, const char *endline) { if (output->type == nmsg_output_type_pres) { @@ -585,6 +621,8 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { } output->stream->type = type; output->stream->buffered = true; + output->stream->so_zmin = NMSG_ZCULL_MIN_WORKERS_DEFAULT; + output->stream->so_zcull = NMSG_ZCULL_SECS_DEFAULT; /* seed the rng, needed for fragment and sequence IDs */ output->stream->random = nmsg_random_init(); diff --git a/nmsg/output.h b/nmsg/output.h index 791203ec..06b67ac8 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -383,40 +383,43 @@ void nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); /** - * Compress containers on a pool of worker threads instead of on the thread - * that filled them. + * Compress containers on worker threads rather than on the thread that filled + * them: a reader that compresses is not reading, and on a busy channel that + * pause is long enough for the socket to overflow. * - * A reader that compresses is not reading, and on a busy channel that pause is - * long enough for the socket to overflow. With a pool the reader hands the - * container over and returns to reading; if every worker is busy it compresses - * the container itself, exactly as it would with no pool, so this is never - * slower than leaving it off. + * workers is a ceiling, not an allocation: threads start on demand and are + * given back once idle, see nmsg_output_set_zlib_cull(). Write order is + * unchanged, and a producer that finds every worker busy compresses inline, so + * this is never slower than leaving it off. * - * Containers are written in the order they were filled, whatever \a workers is - * set to. With a single writing thread that makes the output byte-identical to - * compressing inline; with several, only the write order is guaranteed. + * A write error surfaces on a later nmsg_output_write(), nmsg_output_flush() or + * nmsg_output_close(). File outputs only, and only when buffered. Not + * thread-safe against a concurrent write on the same output. * - * \a workers is a ceiling rather than an allocation: threads are started as - * load calls for them, so an output that never saturates never pays for them. - * The count is capped at an internal limit. - * - * Because a container is written after nmsg_output_write() returns, a write - * error is reported by a later nmsg_output_write(), or by nmsg_output_flush() - * or nmsg_output_close(), rather than by the call that supplied the data. + * \param[in] output nmsg_output_t object. * - * File outputs only, and only when buffered. + * \param[in] workers Maximum number of compressor threads, or 0 to compress + * inline (the default). + */ +void +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); + +/** + * Set when the compressor pool gives threads back. * - * Not thread-safe against a concurrent nmsg_output_write() on the same output: - * call it before the first write, or while no write is in flight. Under - * nmsg_io that is guaranteed, since a close event excludes writers. + * Work goes to the most recently used compressor, so the rest of a pool that + * grew for a burst falls quiet and is culled. Write order is unaffected. May be + * called before or after nmsg_output_set_zlib_workers(). File outputs only. * * \param[in] output nmsg_output_t object. * - * \param[in] workers Maximum number of compressor threads, or 0 to compress - * inline (the default). More than one is only useful when a single output - * is offered more data than one core can compress. + * \param[in] min_workers Compressors culling leaves alone, 1 by default; 0 lets + * the pool empty. A floor on culling, not a number of threads to start. + * + * \param[in] idle_secs Idle seconds that cost a compressor its place, 300 by + * default. 0 disables culling. */ void -nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, unsigned idle_secs); #endif /* NMSG_OUTPUT_H */ diff --git a/nmsg/output_async.c b/nmsg/output_async.c index 0d9b0d2e..ce610f53 100644 --- a/nmsg/output_async.c +++ b/nmsg/output_async.c @@ -40,7 +40,8 @@ * * A producer that finds every worker busy compresses inline rather than wait, * so the pool is never slower than no pool. Workers spawn on demand, making - * 'nworkers' a ceiling rather than an allocation. + * 'nworkers' a ceiling rather than an allocation, and go away again once they + * have been idle long enough; see async_worker(). */ /* @@ -69,9 +70,27 @@ struct async_slot { nmsg_res res; }; +/* + * A compressor. The record outlives the thread: a culled worker leaves its + * 'tid' for the next producer that needs one to join and take over. Idle + * workers are linked most recently used first and a producer takes the head; + * If wakeups are spread evenly instead then nothing is ever idle long enough to cull. + */ +struct async_worker { + pthread_t tid; + pthread_cond_t ready; /* Has a slot, or should look again. */ + struct async_slot *slot; /* Work handed over, or NULL. */ + struct async_worker *idle_prev; + struct async_worker *idle_next; + bool idle; /* On the idle list. */ + bool joinable; /* tid is valid and unjoined. */ + bool exited; /* Thread returned; needs a join. */ + bool reaping; /* A producer is joining it. */ + struct nmsg_ostr_async *pool; +}; + struct nmsg_ostr_async { pthread_mutex_t lock; - pthread_cond_t work_ready; /* A slot became slot_work. */ pthread_cond_t commit_ready; /* The committer's slot is ready. */ pthread_cond_t slot_free; /* A slot became slot_empty. */ /* @@ -79,23 +98,29 @@ struct nmsg_ostr_async { * depth) == i, so a producer runs at most 'depth' tickets ahead of * commit_next and waits only for ticket T - depth to be written. */ - struct async_slot *slots; - unsigned depth; - unsigned nworkers; /* Ceiling; workers start on demand. */ - unsigned nstarted; /* Workers that exist and must be joined. */ - unsigned busy; /* Containers assigned to workers. */ - uint64_t issued; /* Highest ticket claimed, plus one. */ - uint64_t commit_next; /* Ticket allowed to write now. */ - bool shutdown; - bool started; /* Committer exists; must be joined. */ - bool failed; /* No committer; stay inline. */ - bool spawn_failed; /* Worker spawn failed; logged once. */ - pthread_t *workers; - pthread_t committer; - nmsg_res first_error; /* Sticky; surfaced by flush. */ - nmsg_output_t output; - uint64_t n_inline; /* Containers a producer compressed. */ - uint64_t n_waited; /* Producers that waited for a slot. */ + struct async_slot *slots; + unsigned depth; + unsigned nworkers; /* Ceiling; workers start on demand. */ + unsigned nlive; /* Workers running now. */ + unsigned npeak; /* Most that ran at once. */ + unsigned min_workers; /* Workers culling leaves alone. */ + unsigned cull_secs; /* Idle seconds before a cull; 0 is off. */ + clockid_t cull_clock; /* The clock 'ready' was built with. */ + struct async_worker *workers; + struct async_worker *idle_head; /* Most recently used. */ + struct async_worker *idle_tail; + uint64_t issued; /* Highest ticket claimed, plus one. */ + uint64_t commit_next; /* Ticket allowed to write now. */ + bool shutdown; + bool started; /* Committer exists; must be joined. */ + bool failed; /* No committer; stay inline. */ + bool spawn_failed; /* Worker spawn failed; stop trying. */ + pthread_t committer; + nmsg_res first_error; /* Sticky; surfaced by flush. */ + nmsg_output_t output; + uint64_t n_inline; /* Containers a producer compressed. */ + uint64_t n_waited; /* Producers that waited for a slot. */ + uint64_t n_culled; /* Workers that gave up their place. */ }; /* @@ -123,6 +148,87 @@ void _output_async_unref(struct nmsg_stream_output *ostr) pthread_mutex_unlock(&ostr->c_lock); } +/* Put a worker at the head of the idle list. Caller holds pool->lock. */ +static void +async_idle_push(struct nmsg_ostr_async *pool, struct async_worker *worker) +{ + assert(!worker->idle); + + worker->idle_prev = NULL; + worker->idle_next = pool->idle_head; + if (pool->idle_head != NULL) + pool->idle_head->idle_prev = worker; + else + pool->idle_tail = worker; + pool->idle_head = worker; + worker->idle = true; +} + +/* Take a worker out of the idle list. Caller holds pool->lock. */ +static void +async_idle_unlink(struct nmsg_ostr_async *pool, struct async_worker *worker) +{ + assert(worker->idle); + + if (worker->idle_prev != NULL) + worker->idle_prev->idle_next = worker->idle_next; + else + pool->idle_head = worker->idle_next; + + if (worker->idle_next != NULL) + worker->idle_next->idle_prev = worker->idle_prev; + else + pool->idle_tail = worker->idle_prev; + + worker->idle_prev = NULL; + worker->idle_next = NULL; + worker->idle = false; +} + +/* + * The compressor to hand the next container to, or NULL if all of them are + * busy. Caller holds pool->lock. + */ +static struct async_worker * +async_idle_pop(struct nmsg_ostr_async *pool) +{ + struct async_worker *worker = pool->idle_head; + + if (worker != NULL) + async_idle_unlink(pool, worker); + + return (worker); +} + +/* + * When a worker idle from now has outstayed its welcome. Read from the clock + * its condvar was built with: mismatch the two and the deadline lands decades + * out, silently ending culling. Caller holds pool->lock. + */ +static void +async_cull_deadline(const struct nmsg_ostr_async *pool, struct timespec *deadline) +{ + clock_gettime(pool->cull_clock, deadline); + deadline->tv_sec += pool->cull_secs; +} + +/* + * The floor a pool of this size can honour. At the ceiling there is nothing + * left to cull, so say so rather than quietly do nothing. + */ +static unsigned +async_min_workers_for(unsigned nworkers, unsigned min_workers) +{ + if (min_workers > nworkers) { + _nmsg_dprintf(2, "%s: floor of %u lowered to the %u compressor(s) " + "this output may run\n", + __func__, min_workers, nworkers); + min_workers = nworkers; + } + + return (min_workers); +} + /* * How many slots a pool of this size needs. * @@ -185,14 +291,50 @@ async_max_workers(void) return ((unsigned)(ncpu - ASYNC_REORDER_MARGIN)); } +/* Apply a cull policy to a running pool. */ +void _output_async_set_cull(struct nmsg_ostr_async *pool, unsigned min_workers, + unsigned idle_secs) +{ + unsigned i; + + pthread_mutex_lock(&pool->lock); + + pool->min_workers = async_min_workers_for(pool->nworkers, min_workers); + pool->cull_secs = idle_secs; + + /* Parked workers are waiting on the policy that has just been replaced. */ + for (i = 0; i < pool->nworkers; i++) + pthread_cond_signal(&pool->workers[i].ready); + + pthread_mutex_unlock(&pool->lock); +} + +/* Worker counts, for tests and diagnostics. Any of the outputs may be NULL. */ +void _output_async_counts(struct nmsg_ostr_async *pool, unsigned *live, unsigned *peak, + uint64_t *culled) +{ + pthread_mutex_lock(&pool->lock); + if (live != NULL) + *live = pool->nlive; + if (peak != NULL) + *peak = pool->npeak; + if (culled != NULL) + *culled = pool->n_culled; + pthread_mutex_unlock(&pool->lock); +} + nmsg_res _output_async_init(nmsg_output_t output, unsigned nworkers) { struct nmsg_stream_output *ostr = output->stream; struct nmsg_ostr_async *pool; nmsg_res res, old_res = nmsg_res_success; - unsigned depth, max_workers; + unsigned depth, max_workers, zmin, zcull; + unsigned i, nconds = 0; bool same_ceiling; + pthread_condattr_t cattr; + pthread_condattr_t *cattrp = NULL; + clockid_t cull_clock = CLOCK_REALTIME; if (nworkers == 0) return (nmsg_res_success); @@ -203,6 +345,10 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) depth = async_depth_for(nworkers); + /* Set before the pool exists, and carried across a replacement. */ + zmin = ostr->so_zmin; + zcull = ostr->so_zcull; + /* * A pool's ceiling cannot change in place, so a different count means * a replacement. Build it before tearing the old one down, so a failed @@ -210,8 +356,11 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) */ same_ceiling = ostr->so_pool != NULL && ostr->so_pool->nworkers == nworkers; - if (same_ceiling) + if (same_ceiling) { + /* Nothing to rebuild, but the cull policy may have moved on. */ + _output_async_set_cull(ostr->so_pool, zmin, zcull); return (nmsg_res_success); + } pool = calloc(1, sizeof(*pool)); if (pool == NULL) @@ -231,15 +380,43 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) if (pthread_mutex_init(&pool->lock, NULL) != 0) goto fail_mutex; - if (pthread_cond_init(&pool->work_ready, NULL) != 0) - goto fail_work_ready; if (pthread_cond_init(&pool->commit_ready, NULL) != 0) goto fail_commit_ready; if (pthread_cond_init(&pool->slot_free, NULL) != 0) goto fail_slot_free; + /* + * One attribute for every worker condvar, so any of them can carry a + * cull deadline. Monotonic, or a stepped wall clock retimes culling. + */ +#ifdef HAVE_PTHREAD_CONDATTR_SETCLOCK + if (pthread_condattr_init(&cattr) == 0) { + if (pthread_condattr_setclock(&cattr, CLOCK_MONOTONIC) == 0) { + cattrp = &cattr; + cull_clock = CLOCK_MONOTONIC; + } else { + pthread_condattr_destroy(&cattr); + } + } +#endif /* HAVE_PTHREAD_CONDATTR_SETCLOCK */ + + for (nconds = 0; nconds < nworkers; nconds++) { + if (pthread_cond_init(&pool->workers[nconds].ready, cattrp) != 0) + break; + pool->workers[nconds].pool = pool; + } + + if (cattrp != NULL) + pthread_condattr_destroy(cattrp); + + if (nconds < nworkers) + goto fail_worker_conds; + pool->depth = depth; pool->nworkers = nworkers; + pool->cull_clock = cull_clock; + pool->min_workers = async_min_workers_for(nworkers, zmin); + pool->cull_secs = zcull; pool->output = output; if (ostr->so_pool != NULL) @@ -264,11 +441,13 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) return (nmsg_res_success); +fail_worker_conds: + for (i = 0; i < nconds; i++) + pthread_cond_destroy(&pool->workers[i].ready); + pthread_cond_destroy(&pool->slot_free); fail_slot_free: pthread_cond_destroy(&pool->commit_ready); fail_commit_ready: - pthread_cond_destroy(&pool->work_ready); -fail_work_ready: pthread_mutex_destroy(&pool->lock); fail_mutex: free(pool->workers); @@ -292,7 +471,8 @@ _output_async_destroy(nmsg_output_t output) struct nmsg_ostr_async *pool = ostr->so_pool; nmsg_res res; bool started; - unsigned i, nstarted; + unsigned i, npeak; + uint64_t n_culled; if (pool == NULL) return (nmsg_res_success); @@ -311,20 +491,28 @@ _output_async_destroy(nmsg_output_t output) pthread_mutex_lock(&pool->lock); pool->shutdown = true; /* Set under the lock: a thread about */ started = pool->started; /* to wait would miss the wakeup. */ - nstarted = pool->nstarted; - pthread_cond_broadcast(&pool->work_ready); + npeak = pool->npeak; + n_culled = pool->n_culled; + for (i = 0; i < pool->nworkers; i++) + pthread_cond_signal(&pool->workers[i].ready); pthread_cond_broadcast(&pool->commit_ready); pthread_cond_broadcast(&pool->slot_free); pthread_mutex_unlock(&pool->lock); - for (i = 0; i < nstarted; i++) - pthread_join(pool->workers[i], NULL); + /* + * 'joinable' is written only by a spawn, and no producer is left to + * spawn, so it is settled. Culled workers are joined here too. + */ + for (i = 0; i < pool->nworkers; i++) { + if (pool->workers[i].joinable) + pthread_join(pool->workers[i].tid, NULL); + } if (started) pthread_join(pool->committer, NULL); - if (pool->n_inline > 0 || pool->n_waited > 0) - _nmsg_dprintf(2, "%s: %u of %u worker(s) started, %u slot(s); %" PRIu64 " container(s) compressed by the reader, %" PRIu64 " wait(s) for a free slot\n", __func__, nstarted, - pool->nworkers, pool->depth, pool->n_inline, pool->n_waited); + if (pool->n_inline > 0 || pool->n_waited > 0 || n_culled > 0) + _nmsg_dprintf(2, "%s: %u of %u worker(s) at once, %u slot(s); %" PRIu64 " container(s) compressed by the reader, %" PRIu64 " wait(s) for a free slot, %" PRIu64 " worker(s) culled\n", __func__, npeak, + pool->nworkers, pool->depth, pool->n_inline, pool->n_waited, n_culled); /* Read after the joins; the threads write it until they exit. */ res = pool->first_error; @@ -334,9 +522,10 @@ _output_async_destroy(nmsg_output_t output) ostr->so_pool_closing = false; pthread_mutex_unlock(&ostr->c_lock); + for (i = 0; i < pool->nworkers; i++) + pthread_cond_destroy(&pool->workers[i].ready); pthread_cond_destroy(&pool->slot_free); pthread_cond_destroy(&pool->commit_ready); - pthread_cond_destroy(&pool->work_ready); pthread_mutex_destroy(&pool->lock); free(pool->workers); free(pool->slots); @@ -373,30 +562,28 @@ async_all_written(const struct nmsg_ostr_async *pool) } /* - * Compressor thread. Takes any slot needing work, in any order: only write - * order matters, and the committer enforces that. + * Compressor thread. Waits to be handed a slot rather than looking for one, so + * the producer decides which worker runs and the rest go quiet. + * + * A worker idle for cull_secs gives up its place, down to min_workers. That + * decision and leaving the idle list are one lock hold, so a producer can never + * hand work to a thread on its way out. */ static void * async_worker(void *arg) { - struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *)arg; + struct async_worker *self = (struct async_worker *)arg; + struct nmsg_ostr_async *pool = self->pool; + bool timed_out = false; pthread_mutex_lock(&pool->lock); for (;;) { - struct async_slot *slot = NULL; + struct async_slot *slot = self->slot; nmsg_container_t co; uint8_t *buf; size_t buf_len; nmsg_res res; - unsigned i; - - for (i = 0; i < pool->depth; i++) { - if (pool->slots[i].state == slot_work) { - slot = &pool->slots[i]; - break; - } - } if (slot == NULL) { /* @@ -406,10 +593,37 @@ async_worker(void *arg) */ if (pool->shutdown) break; - pthread_cond_wait(&pool->work_ready, &pool->lock); + + if (timed_out && pool->nlive > pool->min_workers) { + pool->n_culled++; + break; + } + + timed_out = false; + + if (!self->idle) + async_idle_push(pool, self); + + /* + * No deadline with culling off, nor at the floor, + * where it could only cost wakeups. The floor is not + * tied to particular threads: grow again and the + * workers added on top are the ones that time out. + */ + if (pool->cull_secs == 0 || + pool->nlive <= pool->min_workers) { + pthread_cond_wait(&self->ready, &pool->lock); + } else { + struct timespec deadline; + + async_cull_deadline(pool, &deadline); + timed_out = pthread_cond_timedwait(&self->ready, + &pool->lock, &deadline) == ETIMEDOUT; + } continue; } + self->slot = NULL; slot->state = slot_taken; co = slot->co; slot->co = NULL; @@ -418,7 +632,7 @@ async_worker(void *arg) res = _output_nmsg_container_compress(pool->output, &co, &buf, &buf_len); pthread_mutex_lock(&pool->lock); - pool->busy--; /* Counted at deposit; see container_submit(). */ + timed_out = false; slot->buf = buf; slot->buf_len = buf_len; slot->res = res; @@ -426,6 +640,11 @@ async_worker(void *arg) pthread_cond_broadcast(&pool->commit_ready); } + if (self->idle) + async_idle_unlink(pool, self); + self->exited = true; + pool->nlive--; + pthread_mutex_unlock(&pool->lock); return (NULL); @@ -532,32 +751,85 @@ async_start(struct nmsg_ostr_async *pool) } /* - * Add a compressor thread, up to the ceiling. Called under pool->lock when a - * producer finds every worker busy, so the pool grows to the load it sees. + * Claim a record for a new compressor, joining the culled thread that left it, + * or NULL when every record is running -- which is what holds the ceiling. * - * Returns false if the thread could not be created: not fatal, the caller - * compresses that container itself. + * The join runs with pool->lock dropped: a returned thread still has the C + * library's teardown to be scheduled for, and waiting under the lock would stop + * the committer writing. Caller holds pool->lock. */ -static bool +static struct async_worker * +async_take_worker(struct nmsg_ostr_async *pool) +{ + struct async_worker *worker = NULL; + unsigned i; + + for (i = 0; i < pool->nworkers; i++) { + /* Free outright: never used, or already reaped. */ + if (!pool->workers[i].joinable) + return (&pool->workers[i]); + + if (worker == NULL && pool->workers[i].exited && + !pool->workers[i].reaping) + worker = &pool->workers[i]; + } + + if (worker == NULL) + return (NULL); + + worker->reaping = true; + pthread_mutex_unlock(&pool->lock); + pthread_join(worker->tid, NULL); + pthread_mutex_lock(&pool->lock); + + /* Cleared before the spawn: if it fails, there is nothing to join. */ + worker->joinable = false; + worker->exited = false; + worker->reaping = false; + + return (worker); +} + +/* + * Add a compressor, up to the ceiling. Called under pool->lock when a producer + * finds no idle worker, so the pool grows to the load it sees. + * + * Returns NULL if one could not be started: not fatal, the caller compresses + * that container itself. + */ +static struct async_worker * async_spawn_worker(struct nmsg_ostr_async *pool) { - int pthread_res; + struct async_worker *worker; + int pthread_res; - pthread_res = pthread_create(&pool->workers[pool->nstarted], NULL, - async_worker, pool); + if (pool->spawn_failed) + return (NULL); + + worker = async_take_worker(pool); + if (worker == NULL) + return (NULL); + + assert(!worker->joinable && !worker->idle && worker->slot == NULL); + + pthread_res = pthread_create(&worker->tid, NULL, async_worker, worker); if (pthread_res != 0) { - /* Logged once; a persistent failure would flood the log. */ - if (!pool->spawn_failed) { - pool->spawn_failed = true; - _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", - __func__, strerror(pthread_res)); - } - return (false); + /* + * Latched, not retried: culling holds the pool below its + * ceiling, so retrying means this syscall per container. + */ + pool->spawn_failed = true; + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", + __func__, strerror(pthread_res)); + return (NULL); } - pool->nstarted++; + worker->joinable = true; + pool->nlive++; + if (pool->nlive > pool->npeak) + pool->npeak = pool->nlive; - return (true); + return (worker); } /* @@ -598,12 +870,12 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, { struct nmsg_stream_output *ostr = output->stream; struct async_slot *slot; + struct async_worker *worker; nmsg_res res = nmsg_res_success; uint8_t *buf; size_t buf_len; bool inline_compress = false; bool committer_needed, pool_usable; - bool worker_idle, below_ceiling; pthread_mutex_lock(&pool->lock); @@ -640,22 +912,24 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, if (ticket >= pool->issued) pool->issued = ticket + 1; - worker_idle = pool->busy < pool->nstarted; - below_ceiling = pool->nstarted < pool->nworkers; - if (is_frag) { /* Only the committer fragments; see async_committer(). */ slot->co = *co; *co = NULL; slot->state = slot_frag; pthread_cond_broadcast(&pool->commit_ready); - } else if (worker_idle || (below_ceiling && async_spawn_worker(pool))) { - /* Hand the container over and get back to reading. */ + } else if ((worker = async_idle_pop(pool)) != NULL || + (worker = async_spawn_worker(pool)) != NULL) { + /* + * Hand the container over and get back to reading. The slot + * stays ours across the spawn's lock drop: no other ticket + * maps to it, and the committer waits until it is ready. + */ slot->co = *co; *co = NULL; slot->state = slot_work; - pool->busy++; - pthread_cond_signal(&pool->work_ready); + worker->slot = slot; + pthread_cond_signal(&worker->ready); } else { /* Everyone busy and at the ceiling: compress here. */ slot->state = slot_taken; diff --git a/nmsg/private.h b/nmsg/private.h index a05bbe95..7a5f3f13 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -312,6 +312,14 @@ struct nmsg_stream_input { */ struct nmsg_ostr_async; +/* + * Compressor threads left alone when the pool shrinks, and the idle time that + * costs a thread its place. Defaults rather than constants: nmsg_output_set_zlib_cull() + * overrides both, and output_open_stream_base() seeds every stream with them. + */ +#define NMSG_ZCULL_MIN_WORKERS_DEFAULT 1 +#define NMSG_ZCULL_SECS_DEFAULT 300 + /* nmsg_stream_output: used by nmsg_output */ struct nmsg_stream_output { pthread_mutex_t c_lock; /* Container lock. */ @@ -341,6 +349,8 @@ struct nmsg_stream_output { bool so_pool_closing; /* Pool teardown started; c_lock. */ pthread_cond_t c_drained; /* so_inflight == 0; c_lock. */ struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ + unsigned so_zmin; /* Compressors culling leaves; c_lock. */ + unsigned so_zcull; /* Idle seconds before a cull; c_lock. */ }; /* nmsg_callback_output: used by nmsg_output */ @@ -604,6 +614,9 @@ nmsg_res _output_nmsg_frag_write(nmsg_output_t, nmsg_container_t); /* from output_async.c */ nmsg_res _output_async_init(nmsg_output_t, unsigned); nmsg_res _output_async_destroy(nmsg_output_t); +void _output_async_set_cull(struct nmsg_ostr_async *, unsigned, unsigned); +void _output_async_counts(struct nmsg_ostr_async *, unsigned *, + unsigned *, uint64_t *); nmsg_res _output_async_drain(struct nmsg_ostr_async *); struct nmsg_ostr_async *_output_async_ref(struct nmsg_stream_output *); void _output_async_unref(struct nmsg_stream_output *); diff --git a/src/nmsgtool.c b/src/nmsgtool.c index d1a65327..12891d6a 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -36,7 +36,14 @@ /* Globals. */ -static nmsgtool_ctx ctx; +/* + * Defaults set here rather than after argv_process(): 0 is a meaningful value + * for both cull options, so neither can use it as an "unset" marker. + */ +static nmsgtool_ctx ctx = { + .zmin = NMSGTOOL_ZMIN_DEFAULT, + .zcull = NMSGTOOL_ZCULL_DEFAULT, +}; static argv_t args[] = { { 'b', "bpf", @@ -335,6 +342,18 @@ static argv_t args[] = { "n", "compress file output on n threads" }, + { '\0', "zcull", + ARGV_INT, + &ctx.zcull, + "secs", + "drop a compressor idle after n secs (0 never)" }, + + { '\0', "zmin", + ARGV_INT, + &ctx.zmin, + "n", + "never drop below n compressors" }, + { ARGV_LAST, 0, 0, 0, 0, 0 } }; @@ -510,16 +529,19 @@ setup_nmsg_output_workers(nmsgtool_ctx *c) { c->zworkers_resolved = zworkers_count(c); if (c->initial_outputs != NULL) { - for (i = 0; i < output_vec_size(c->initial_outputs); i++) - nmsg_output_set_zlib_workers( - output_vec_data(c->initial_outputs)[i], - c->zworkers_resolved); + for (i = 0; i < output_vec_size(c->initial_outputs); i++) { + nmsg_output_t output = output_vec_data(c->initial_outputs)[i]; + + nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); + nmsg_output_set_zlib_workers(output, c->zworkers_resolved); + } output_vec_destroy(&c->initial_outputs); } if (c->zworkers_resolved > 0 && c->debug >= 2) - fprintf(stderr, "%s: compressing on up to %u thread(s) per output\n", - argv_program, c->zworkers_resolved); + fprintf(stderr, "%s: compressing on up to %u thread(s) per output, " + "keeping %d idle for %d second(s)\n", argv_program, + c->zworkers_resolved, c->zmin, c->zcull); } void @@ -527,6 +549,8 @@ setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_buffered(output, !(c->unbuffered)); nmsg_output_set_endline(output, c->endline_str); nmsg_output_set_zlibout(output, c->zlibout); + /* Before the ceiling: setting that builds the pool, which reads this. */ + nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); nmsg_output_set_zlib_workers(output, c->zworkers_resolved); nmsg_output_set_source(output, c->set_source); nmsg_output_set_operator(output, c->set_operator); diff --git a/src/nmsgtool.h b/src/nmsgtool.h index baad358e..bc86b448 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -66,6 +66,13 @@ VECTOR_GENERATE(output_vec, nmsg_output_t) */ #define NMSGTOOL_ZWORKERS_MAX(ncpu) (ncpu) +/* + * Cull policy defaults. NMSGTOOL_ZMIN_DEFAULT is a floor on the live thread + * count, unrelated to NMSGTOOL_ZWORKERS_MIN above, which floors the ceiling. + */ +#define NMSGTOOL_ZMIN_DEFAULT 1 +#define NMSGTOOL_ZCULL_DEFAULT 300 + typedef struct { /* parameters */ argv_array_t filters, statsmods; @@ -74,6 +81,8 @@ typedef struct { argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; int zasync; /* Compressor threads; -1 chooses. */ + int zmin; /* Compressors culling leaves. */ + int zcull; /* Idle seconds before a cull. */ char *endline, *kicker, *mname, *vname, *bpfstr, *filter_policy, *kafka_key_field; int debug, signal; unsigned mtu, count, interval, rate, freq, byte_rate; diff --git a/src/process_args.c b/src/process_args.c index 2150cde6..08b5ec8f 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -129,6 +129,18 @@ process_args(nmsgtool_ctx *c) { usage("--zasync must be -1 (choose), 0 (off), " "or a thread count no greater than the available cores"); + if (c->zcull < 0) + usage("--zcull must be 0 (never cull) or a number of seconds"); + + /* + * Only checked for sanity here: the ceiling --zmin is a floor under is + * not settled until setup_nmsg_output_workers(), and libnmsg lowers it + * again to what the pool can run. -dd reports what it ended up as. + */ + if (c->zmin < 0 || c->zmin > NMSGTOOL_ZWORKERS_MAX(nmsgtool_ncpu())) + usage("--zmin must be 0 or a thread count no greater than " + "the available cores"); + if (c->vname == NULL && c->mname != NULL) c->vname = "base"; diff --git a/tests/test-zpool-cull.c b/tests/test-zpool-cull.c new file mode 100644 index 00000000..04d19492 --- /dev/null +++ b/tests/test-zpool-cull.c @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Compressors have to give their place back once they go idle, and the pool has + * to grow again afterwards. Linked against libnmsg's own objects, since the + * live worker count is not something the public headers expose. + * + * Only the first spawn is deterministic: a producer that finds no idle worker + * starts one, so a single write guarantees exactly one. Growing past that needs + * production to outrun compression, so nothing here asserts on it. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" +#include "private.h" + +#define BUFSZ NMSG_WBUFSZ_JUMBO +#define CULL_SECS 1 + +/* Long enough that a CULL_SECS deadline has certainly passed. */ +#define SETTLE_SECS 3 + +static nmsg_msgmod_t mod; + +/* automake has no per-test timeout, so a wedged pool would hang forever. */ +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-cull: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* Nothing useful to do; we are on our way out regardless. */ + } + _exit(1); +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-cull: %s\n", what); + exit(1); +} + +static void +fail_count(const char *what, unsigned want, unsigned got) +{ + fprintf(stderr, "test-zpool-cull: %s: wanted %u, got %u\n", what, want, got); + exit(1); +} + +static nmsg_message_t +make_message(unsigned i) +{ + char payload[48]; + nmsg_message_t msg; + size_t len; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + len = snprintf(payload, sizeof(payload), "payload %u", i); + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + return (msg); +} + +static unsigned +count_payloads(const char *path) +{ + nmsg_input_t input; + nmsg_message_t msg; + unsigned n = 0; + int fd; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + while (nmsg_input_read(input, &msg) == nmsg_res_success) { + nmsg_message_destroy(&msg); + n += 1; + } + + nmsg_input_close(&input); + close(fd); + + return (n); +} + +static nmsg_output_t +open_output(const char *path, unsigned workers, unsigned zmin, unsigned zcull) +{ + nmsg_output_t output; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + nmsg_output_set_zlib_cull(output, zmin, zcull); + nmsg_output_set_zlib_workers(output, workers); + + return (output); +} + +/* Write 'n' payloads and seal what they filled, so the pool sees a container. */ +static void +write_burst(nmsg_output_t output, unsigned n, unsigned tag) +{ + unsigned i; + + for (i = 0; i < n; i++) { + nmsg_message_t msg = make_message(tag + i); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + } + + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); +} + +static void +counts(nmsg_output_t output, unsigned *live, unsigned *peak, uint64_t *culled) +{ + struct nmsg_ostr_async *pool = output->stream->so_pool; + + if (pool == NULL) + fail("output has no compressor pool"); + + _output_async_counts(pool, live, peak, culled); +} + +/* + * A worker starts, goes quiet and gives its place back; the pool then grows + * again from empty. Reaching zero is what exercises reusing a culled worker's + * record, which means joining the thread that left it. + */ +static void +test_cull_to_empty(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, CULL_SECS); + unsigned live, peak; + uint64_t culled; + + write_burst(output, 100, 0); + counts(output, &live, &peak, &culled); + if (live != 1) + fail_count("worker not started", 1, live); + + sleep(SETTLE_SECS); + + counts(output, &live, &peak, &culled); + if (live != 0) + fail_count("pool did not empty", 0, live); + if (culled != 1) + fail_count("culls recorded", 1, (unsigned) culled); + + /* Growing again has to reuse the record the culled worker left. */ + write_burst(output, 100, 100); + counts(output, &live, &peak, &culled); + if (live != 1) + fail_count("pool did not grow again", 1, live); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + if (count_payloads(path) != 200) + fail_count("payloads written", 200, count_payloads(path)); +} + +/* The floor is left alone, however long the pool stays quiet. */ +static void +test_floor(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 1, CULL_SECS); + unsigned live; + uint64_t culled; + + write_burst(output, 100, 0); + sleep(SETTLE_SECS); + + counts(output, &live, NULL, &culled); + if (live != 1) + fail_count("floor not held", 1, live); + if (culled != 0) + fail_count("culls below the floor", 0, (unsigned) culled); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* Culling off leaves the pool at its high water mark. */ +static void +test_cull_disabled(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, 0); + unsigned live, peak; + uint64_t culled; + + write_burst(output, 100, 0); + counts(output, &live, &peak, &culled); + if (live != peak) + fail_count("workers lost before settling", peak, live); + + sleep(SETTLE_SECS); + + counts(output, &live, &peak, &culled); + if (live != peak) + fail_count("culled with culling disabled", peak, live); + if (culled != 0) + fail_count("culls with culling disabled", 0, (unsigned) culled); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* + * A floor above the ceiling has nothing to cull, and must be lowered to it + * rather than quietly disable the policy. + */ +static void +test_floor_above_ceiling(const char *path) +{ + nmsg_output_t output = open_output(path, 1, 8, CULL_SECS); + unsigned live; + + write_burst(output, 100, 0); + sleep(SETTLE_SECS); + + counts(output, &live, NULL, NULL); + if (live != 1) + fail_count("clamped floor not held", 1, live); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* + * Work keeps flowing across culls: containers written while the pool is + * shrinking and regrowing all have to arrive, in order. + */ +static void +test_traffic_across_culls(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, CULL_SECS); + unsigned round, live; + + for (round = 0; round < 3; round++) { + write_burst(output, 100, round * 100); + sleep(SETTLE_SECS); + } + + counts(output, &live, NULL, NULL); + if (live != 0) + fail_count("pool did not settle", 0, live); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + if (count_payloads(path) != 300) + fail_count("payloads written", 300, count_payloads(path)); +} + +int main(void) { + char path[] = "/tmp/nmsg-zpool-cull.XXXXXX"; + int fd; + + signal(SIGALRM, on_alarm); + alarm(120); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + /* mkstemp() only to get a unique name; the writers reopen by path. */ + fd = mkstemp(path); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + + test_cull_to_empty(path); + test_floor(path); + test_cull_disabled(path); + test_floor_above_ceiling(path); + test_traffic_across_culls(path); + + unlink(path); + + return (0); +} From bdc19ef5d85c527d7880cc44adcc6aac280481d7 Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Fri, 21 Aug 2026 14:54:49 -0400 Subject: [PATCH 17/18] Fix use-after-free on so_pool and a message lost on a pool error; pick workers by index --- .gitignore | 2 - Makefile.am | 17 +- doc/docbook/nmsgtool.docbook | 70 ++++--- libmy/my_cpu.h | 31 +++ nmsg/output.c | 51 +++-- nmsg/output.h | 21 +- nmsg/output_async.c | 391 +++++++++++++++++------------------ nmsg/output_nmsg.c | 26 ++- nmsg/private.h | 17 +- src/nmsgtool.c | 114 ++++++---- src/nmsgtool.h | 2 +- src/process_args.c | 20 +- tests/test-zpool-cull.c | 134 +++++++++--- tests/test-zpool-mt.c | 43 +++- tests/test-zpool-order.c | 72 +++++-- 15 files changed, 615 insertions(+), 396 deletions(-) create mode 100644 libmy/my_cpu.h diff --git a/.gitignore b/.gitignore index 158f87fe..edea6ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -72,5 +72,3 @@ tests/testzmq.json tests/testzmq.sock tests/*/*.out tests/*/test.sh -.clang-format -.clangd diff --git a/Makefile.am b/Makefile.am index f459a305..aa62d1d4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -216,8 +216,8 @@ LIBNMSG_LIB_MODULES = \ nmsg/msgmodset.c \ nmsg/nmsg.c \ nmsg/output.c \ - nmsg/output_json.c \ nmsg/output_async.c \ + nmsg/output_json.c \ nmsg/output_nmsg.c \ nmsg/output_pres.c \ nmsg/payload.c \ @@ -245,6 +245,7 @@ LIBNMSG_LIB_MODULES = \ nmsg_libnmsg_la_SOURCES = \ libmy/crc32c.h \ libmy/list.h \ + libmy/my_cpu.h \ libmy/my_time.h \ libmy/my_rate.h \ libmy/tree.h \ @@ -420,6 +421,7 @@ src_nmsgtool_SOURCES = \ libmy/argv.c \ libmy/argv.h \ libmy/argv_loc.h \ + libmy/my_cpu.h \ src/daemon.c \ src/getsock.c \ src/io.c \ @@ -437,6 +439,11 @@ src_nmsgtool_SOURCES = \ ## # +# Tests that reach private symbols link the objects: libnmsg.la exports +# nmsg_* only, and the dlopened base msgmod resolves its own from the +# executable, which is what -rdynamic is for. +PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o) + TESTS_ENVIRONMENT = NMSG_MSGMOD_DIR=$(abs_top_builddir)/nmsg/base/.libs TESTS_ENVIRONMENT += abs_top_builddir='$(abs_top_builddir)' abs_top_srcdir='$(abs_top_srcdir)' @@ -538,7 +545,6 @@ TESTS += tests/test-private check_PROGRAMS += tests/test-private tests_test_private_LDFLAGS = -rdynamic tests_test_private_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS) -PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o) tests_test_private_LDADD = \ $(PRIVATE_TEST_MODULES) \ nmsg/nmsg.pb-c.o \ @@ -564,8 +570,12 @@ tests_test_nmsg_output_set_rate_LDADD = nmsg/libnmsg.la TESTS += tests/test-zpool-order check_PROGRAMS += tests/test-zpool-order +tests_test_zpool_order_LDFLAGS = -rdynamic +tests_test_zpool_order_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) tests_test_zpool_order_SOURCES = tests/test-zpool-order.c -tests_test_zpool_order_LDADD = nmsg/libnmsg.la TESTS += tests/test-zpool-mt check_PROGRAMS += tests/test-zpool-mt @@ -575,7 +585,6 @@ tests_test_zpool_mt_LDADD = nmsg/libnmsg.la TESTS += tests/test-zpool-cull check_PROGRAMS += tests/test-zpool-cull tests_test_zpool_cull_LDFLAGS = -rdynamic -tests_test_zpool_cull_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS) tests_test_zpool_cull_LDADD = \ $(PRIVATE_TEST_MODULES) \ nmsg/nmsg.pb-c.o \ diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 75dfa34e..16163bf5 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -667,28 +667,29 @@ overflow; handing the container to a compressor lets the reader carry straight on. - n is a ceiling rather than - a thread count: compressors are started only as load calls - for them, and given back once they go idle, so an output - that never saturates never pays for them. See - and . Compression is CPU-bound, so a thread per core is as - far as it can help: values above the number of cores - available to the process are rejected, and libnmsg applies - the same rule slightly tighter, since the buffer that feeds - the compressors is sized from their number. - - A value of chooses the ceiling: - two per input, since a reader thread can saturate roughly - one core compressing, bounded by the cores the readers leave - spare, never fewer than four because a single input can - carry several cores' worth on its own, and then divided - across the file outputs, which each get their own pool and - share the same cores. Sizing it exactly is not important, - because a reader that finds every compressor busy compresses - the container itself, exactly as it would with this option - unset. Too small a pool therefore degrades to the behaviour - of no pool at all rather than stalling, and neither setting - is slower than leaving the option off. + Off by default. n is a + ceiling rather than a thread count: compressors start only as + load calls for them and are given back once idle, so an + output that never saturates never pays for them. See + and . + Compression is CPU-bound, so a thread per core is as far as + it can help; values above the cores available to the process + are rejected. + + A value of chooses the ceiling: two + per input, split across the file outputs, which each get + their own pool and share the same cores, then bounded by the + cores the readers leave spare. An output gets at least four + regardless, since a single input can carry several cores' + worth on its own, and never more than the core count. Sizing + it exactly is not important: a reader that finds every + compressor busy compresses the container itself, so too small + a pool costs nothing beyond the pool being idle. + + The cores counted are those in the process affinity + mask, which does not reflect a cgroup CPU quota; under a + container runtime that caps CPU rather than pinning it, pass + n explicitly. This is worth using when a single input carries more data than one core can compress, and on file-to-file work @@ -701,9 +702,15 @@ identical to compressing inline, whatever n is. + Compression itself still needs ; + without it a compressor only takes the serializing off the + reader. A pool holds a serialized container per queued + ticket, so it can add tens of megabytes of buffering to an + output. + Containers are always written in the order they were filled. Applies to file () outputs only, - and has no effect with + and cannot be combined with . @@ -716,12 +723,12 @@ value of never gives one up, leaving a pool at the largest it ever needed to be. - Containers go to the compressor used most recently, so - a pool that grew for a burst keeps the front of that order - busy and lets the rest fall quiet. The pool grows again on - demand, exactly as it did the first time, and a compressor - only ever leaves when it is holding nothing, so the output - is unaffected either way. + Containers always go to the lowest-numbered free + compressor, so a pool that grew for a burst keeps the first + few busy and lets the rest fall quiet. The pool grows again + on demand, exactly as it did the first time, and a + compressor only ever leaves when it is holding nothing, so + the output is unaffected either way. This matters mainly for a long-running output that is not being rotated: an output closed by @@ -742,8 +749,9 @@ number of threads to start: compressors are still started only on demand, so an output that has never been busy is running none of them whatever this is set to. A value above - the ceiling in force is lowered to it, which is reported at - . + an explicit is rejected; under + it is lowered to the ceiling + chosen, which is reported at . One thread per file output is not culled in any case: the thread that does the writing is not a compressor. diff --git a/libmy/my_cpu.h b/libmy/my_cpu.h new file mode 100644 index 00000000..8f789ddf --- /dev/null +++ b/libmy/my_cpu.h @@ -0,0 +1,31 @@ +#ifndef MY_CPU_H +#define MY_CPU_H + +#include + +#ifdef __linux__ +# include +#endif /* __linux__ */ + +/* Cores this process may actually run on. Never less than one. */ +static inline long +my_ncpu(void) +{ + long ncpu = -1; + +#ifdef __linux__ + cpu_set_t set; + + if (sched_getaffinity(0, sizeof(set), &set) == 0) + ncpu = CPU_COUNT(&set); +#endif /* __linux__ */ + + if (ncpu < 1) + ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if (ncpu < 1) + ncpu = 1; + + return (ncpu); +} + +#endif /* MY_CPU_H */ diff --git a/nmsg/output.c b/nmsg/output.c index 0e062fca..19c499a3 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -419,48 +419,36 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { output->stream->do_zlib = zlibout; } -void -nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) { - nmsg_res res; - +nmsg_res +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) +{ /* * Type test first: 'stream' is a union member, so reading stream->type * on a pres or json output would reinterpret another struct's bytes. */ if (output->type != nmsg_output_type_stream) - return; + return (nmsg_res_success); if (output->stream->type != nmsg_stream_type_file) - return; + return (nmsg_res_success); /* * Unbuffered flushes a container per message, so a pool would spend a * ticket and a wakeup per message to compress a single payload. */ - if (workers > 0 && !output->stream->buffered) { - _nmsg_dprintf(1, "%s: ignored: not available on unbuffered output\n", - __func__); - return; - } + if (workers > 0 && !output->stream->buffered) + return (nmsg_res_failure); - /* - * Nothing to return an error through, so both paths log: disabling a - * pool after writing can strand an error a worker recorded. - */ - if (workers > 0) { - res = _output_async_init(output, workers); - if (res != nmsg_res_success) - _nmsg_dprintf(1, "%s: could not start compressor: %s\n", - __func__, nmsg_res_lookup(res)); - } else { - res = _output_async_destroy(output); - if (res != nmsg_res_success) - _nmsg_dprintf(1, "%s: compressor reported: %s\n", - __func__, nmsg_res_lookup(res)); - } + if (workers > 0) + return (_output_async_init(output, workers)); + + /* Turning a pool off can strand an error a worker recorded. */ + return (_output_async_destroy(output)); } void -nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, unsigned idle_secs) { +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, + unsigned idle_secs) +{ struct nmsg_stream_output *ostr; struct nmsg_ostr_async *pool; @@ -634,7 +622,14 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { pthread_mutex_init(&output->stream->c_lock, NULL); pthread_mutex_init(&output->stream->w_lock, NULL); - pthread_cond_init(&output->stream->c_drained, NULL); + if (pthread_cond_init(&output->stream->c_drained, NULL) != 0) { + nmsg_random_destroy(&output->stream->random); + pthread_mutex_destroy(&output->stream->c_lock); + pthread_mutex_destroy(&output->stream->w_lock); + free(output->stream); + free(output); + return (NULL); + } /* * Enable container sequencing. Sock and zmq only, which is what lets diff --git a/nmsg/output.h b/nmsg/output.h index 06b67ac8..18e0a4af 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -393,23 +393,29 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); * this is never slower than leaving it off. * * A write error surfaces on a later nmsg_output_write(), nmsg_output_flush() or - * nmsg_output_close(). File outputs only, and only when buffered. Not - * thread-safe against a concurrent write on the same output. + * nmsg_output_close(). File outputs only, and only when buffered. + * + * Not thread-safe against a concurrent write on the same output: turning a pool + * on or off while another thread is writing can reorder containers. * * \param[in] output nmsg_output_t object. * * \param[in] workers Maximum number of compressor threads, or 0 to compress * inline (the default). + * + * \return #nmsg_res_success, or #nmsg_res_failure on an unbuffered output. + * Setting 0 returns any write error the pool had yet to report. */ -void +nmsg_res nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); /** * Set when the compressor pool gives threads back. * - * Work goes to the most recently used compressor, so the rest of a pool that - * grew for a burst falls quiet and is culled. Write order is unaffected. May be - * called before or after nmsg_output_set_zlib_workers(). File outputs only. + * Work always goes to the lowest-numbered free compressor, so the rest of a + * pool that grew for a burst falls quiet and is culled. Write order is + * unaffected. May be called before or after nmsg_output_set_zlib_workers(). + * File outputs only. * * \param[in] output nmsg_output_t object. * @@ -420,6 +426,7 @@ nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); * default. 0 disables culling. */ void -nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, unsigned idle_secs); +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, + unsigned idle_secs); #endif /* NMSG_OUTPUT_H */ diff --git a/nmsg/output_async.c b/nmsg/output_async.c index ce610f53..8564f16b 100644 --- a/nmsg/output_async.c +++ b/nmsg/output_async.c @@ -24,9 +24,7 @@ #include "private.h" -#ifdef __linux__ -#include -#endif /* __linux__ */ +#include "libmy/my_cpu.h" /* Data structures. */ @@ -72,20 +70,17 @@ struct async_slot { /* * A compressor. The record outlives the thread: a culled worker leaves its - * 'tid' for the next producer that needs one to join and take over. Idle - * workers are linked most recently used first and a producer takes the head; - * If wakeups are spread evenly instead then nothing is ever idle long enough to cull. + * 'tid' for the committer to join before the record is reused. Producers always + * take the lowest free index, so the front of the array carries the load; + * spread the work evenly instead and nothing is idle long enough to cull. */ struct async_worker { pthread_t tid; - pthread_cond_t ready; /* Has a slot, or should look again. */ - struct async_slot *slot; /* Work handed over, or NULL. */ - struct async_worker *idle_prev; - struct async_worker *idle_next; - bool idle; /* On the idle list. */ + pthread_cond_t ready; /* Has a slot, or should look again. */ + struct async_slot *slot; /* Work handed over, or NULL. */ + bool idle; /* Waiting for work. */ bool joinable; /* tid is valid and unjoined. */ bool exited; /* Thread returned; needs a join. */ - bool reaping; /* A producer is joining it. */ struct nmsg_ostr_async *pool; }; @@ -107,8 +102,6 @@ struct nmsg_ostr_async { unsigned cull_secs; /* Idle seconds before a cull; 0 is off. */ clockid_t cull_clock; /* The clock 'ready' was built with. */ struct async_worker *workers; - struct async_worker *idle_head; /* Most recently used. */ - struct async_worker *idle_tail; uint64_t issued; /* Highest ticket claimed, plus one. */ uint64_t commit_next; /* Ticket allowed to write now. */ bool shutdown; @@ -148,68 +141,42 @@ void _output_async_unref(struct nmsg_stream_output *ostr) pthread_mutex_unlock(&ostr->c_lock); } -/* Put a worker at the head of the idle list. Caller holds pool->lock. */ -static void -async_idle_push(struct nmsg_ostr_async *pool, struct async_worker *worker) -{ - assert(!worker->idle); - - worker->idle_prev = NULL; - worker->idle_next = pool->idle_head; - if (pool->idle_head != NULL) - pool->idle_head->idle_prev = worker; - else - pool->idle_tail = worker; - pool->idle_head = worker; - worker->idle = true; -} - -/* Take a worker out of the idle list. Caller holds pool->lock. */ -static void -async_idle_unlink(struct nmsg_ostr_async *pool, struct async_worker *worker) -{ - assert(worker->idle); - - if (worker->idle_prev != NULL) - worker->idle_prev->idle_next = worker->idle_next; - else - pool->idle_head = worker->idle_next; - - if (worker->idle_next != NULL) - worker->idle_next->idle_prev = worker->idle_prev; - else - pool->idle_tail = worker->idle_prev; - - worker->idle_prev = NULL; - worker->idle_next = NULL; - worker->idle = false; -} - /* * The compressor to hand the next container to, or NULL if all of them are - * busy. Caller holds pool->lock. + * busy. Lowest free index every time, so the front of the array takes the load + * and the back stays idle long enough to be worth culling. + * + * Taken in the same lock hold that hands it a slot, so a producer can never + * pick one that is on its way out. Caller holds pool->lock. */ static struct async_worker * -async_idle_pop(struct nmsg_ostr_async *pool) +async_idle_take(struct nmsg_ostr_async *pool) { - struct async_worker *worker = pool->idle_head; + unsigned i; - if (worker != NULL) - async_idle_unlink(pool, worker); + for (i = 0; i < pool->nworkers; i++) { + if (pool->workers[i].idle) { + pool->workers[i].idle = false; + return (&pool->workers[i]); + } + } - return (worker); + return (NULL); } /* * When a worker idle from now has outstayed its welcome. Read from the clock - * its condvar was built with: mismatch the two and the deadline lands decades - * out, silently ending culling. Caller holds pool->lock. + * its condvar was built with. */ -static void +static bool async_cull_deadline(const struct nmsg_ostr_async *pool, struct timespec *deadline) { - clock_gettime(pool->cull_clock, deadline); + if (clock_gettime(pool->cull_clock, deadline) != 0) + return (false); + deadline->tv_sec += pool->cull_secs; + + return (true); } /* @@ -237,9 +204,10 @@ async_min_workers_for(unsigned nworkers, unsigned min_workers) * and what it would be covering for is a slow write -- which is single-threaded * on the committer, and no amount of depth helps. * - * Sized here rather than fixed because the worker count spans an order of - * magnitude across the boxes this runs on, and a buffer sized for the largest - * is megabytes that a two-worker output never touches. + * Sized here rather than fixed because a slot pins a compressed container until + * every earlier ticket is written: at the 1 MiB containers a file output uses, + * a buffer sized for the largest box is tens of megabytes that a two-worker + * output never needs. */ static unsigned async_depth_for(unsigned nworkers) @@ -249,29 +217,6 @@ async_depth_for(unsigned nworkers) return (depth < ASYNC_DEPTH_MIN ? ASYNC_DEPTH_MIN : depth); } -/* - * Cores this process may run on. nmsgtool computes the same thing for its own - * sizing, but sees only the public header and cannot reach this one. - */ -static long -async_ncpu(void) -{ - long ncpu = -1; -#ifdef __linux__ - cpu_set_t set; - - if (sched_getaffinity(0, sizeof(set), &set) == 0) - ncpu = CPU_COUNT(&set); -#endif /* __linux__ */ - - if (ncpu < 1) - ncpu = sysconf(_SC_NPROCESSORS_ONLN); - if (ncpu < 1) - ncpu = 1; - - return (ncpu); -} - /* * Compressor threads one output may have. Compression is CPU-bound, so more * than one thread per core cannot help; past that they only add context @@ -283,12 +228,7 @@ async_ncpu(void) static unsigned async_max_workers(void) { - long ncpu = async_ncpu(); - - if (ncpu < ASYNC_DEPTH_MIN) - ncpu = ASYNC_DEPTH_MIN; - - return ((unsigned)(ncpu - ASYNC_REORDER_MARGIN)); + return ((unsigned)my_ncpu()); } /* Apply a cull policy to a running pool. */ @@ -310,8 +250,8 @@ void _output_async_set_cull(struct nmsg_ostr_async *pool, unsigned min_workers, } /* Worker counts, for tests and diagnostics. Any of the outputs may be NULL. */ -void _output_async_counts(struct nmsg_ostr_async *pool, unsigned *live, unsigned *peak, - uint64_t *culled) +void _output_async_counts(struct nmsg_ostr_async *pool, unsigned *live, + unsigned *peak, uint64_t *culled) { pthread_mutex_lock(&pool->lock); if (live != NULL) @@ -327,7 +267,7 @@ nmsg_res _output_async_init(nmsg_output_t output, unsigned nworkers) { struct nmsg_stream_output *ostr = output->stream; - struct nmsg_ostr_async *pool; + struct nmsg_ostr_async *pool, *old; nmsg_res res, old_res = nmsg_res_success; unsigned depth, max_workers, zmin, zcull; unsigned i, nconds = 0; @@ -345,23 +285,33 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) depth = async_depth_for(nworkers); - /* Set before the pool exists, and carried across a replacement. */ + /* + * The policy is set before the pool exists and carried across a + * replacement. Read with the pool under one c_lock hold, and the pool + * referenced, so a teardown cannot free it underneath. + */ + pthread_mutex_lock(&ostr->c_lock); zmin = ostr->so_zmin; zcull = ostr->so_zcull; + old = _output_async_ref(ostr); + same_ceiling = old != NULL && old->nworkers == nworkers; + pthread_mutex_unlock(&ostr->c_lock); /* * A pool's ceiling cannot change in place, so a different count means * a replacement. Build it before tearing the old one down, so a failed * allocation leaves the working pool in place. */ - same_ceiling = ostr->so_pool != NULL && - ostr->so_pool->nworkers == nworkers; if (same_ceiling) { /* Nothing to rebuild, but the cull policy may have moved on. */ - _output_async_set_cull(ostr->so_pool, zmin, zcull); + _output_async_set_cull(old, zmin, zcull); + _output_async_unref(ostr); return (nmsg_res_success); } + if (old != NULL) + _output_async_unref(ostr); + pool = calloc(1, sizeof(*pool)); if (pool == NULL) return (nmsg_res_memfail); @@ -419,8 +369,8 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) pool->cull_secs = zcull; pool->output = output; - if (ostr->so_pool != NULL) - old_res = _output_async_destroy(output); + /* A no-op under c_lock if there is nothing to replace. */ + old_res = _output_async_destroy(output); pthread_mutex_lock(&ostr->c_lock); @@ -428,8 +378,13 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) * Tickets span the stream, not the pool, so a pool built mid-stream * must start where the stream got to. Seeded and published in one * c_lock hold, so a ticket either predates the pool or is at or above - * commit_next -- never below, where the committer would wait for it + * commit_next, never below, where the committer would wait for it * forever. + * + * That settles the pool's own liveness, not the byte order: a container + * from a ticket just before this may still be waiting to go out inline, + * and nothing holds this pool back for it. See + * nmsg_output_set_zlib_workers(). */ pool->commit_next = pool->issued = ostr->so_ticket; @@ -468,21 +423,33 @@ nmsg_res _output_async_destroy(nmsg_output_t output) { struct nmsg_stream_output *ostr = output->stream; - struct nmsg_ostr_async *pool = ostr->so_pool; + struct nmsg_ostr_async *pool; nmsg_res res; bool started; unsigned i, npeak; - uint64_t n_culled; + uint64_t n_culled, n_inline, n_waited; - if (pool == NULL) + pthread_mutex_lock(&ostr->c_lock); + + /* + * One teardown at a time. A second waits the first out rather than + * returning, so a close cannot reach the fd while another thread's + * committer is still draining onto it. + */ + while (ostr->so_pool_closing) + pthread_cond_wait(&ostr->c_drained, &ostr->c_lock); + + pool = ostr->so_pool; + if (pool == NULL) { + pthread_mutex_unlock(&ostr->c_lock); return (nmsg_res_success); + } /* * Stop issuing tickets to the pool, then wait for the outstanding ones * to arrive. Both under c_lock, which orders them against issuance: * once this returns, no producer is still en route with a ticket. */ - pthread_mutex_lock(&ostr->c_lock); ostr->so_pool_closing = true; while (ostr->so_inflight > 0) pthread_cond_wait(&ostr->c_drained, &ostr->c_lock); @@ -493,6 +460,8 @@ _output_async_destroy(nmsg_output_t output) started = pool->started; /* to wait would miss the wakeup. */ npeak = pool->npeak; n_culled = pool->n_culled; + n_inline = pool->n_inline; + n_waited = pool->n_waited; for (i = 0; i < pool->nworkers; i++) pthread_cond_signal(&pool->workers[i].ready); pthread_cond_broadcast(&pool->commit_ready); @@ -500,19 +469,25 @@ _output_async_destroy(nmsg_output_t output) pthread_mutex_unlock(&pool->lock); /* - * 'joinable' is written only by a spawn, and no producer is left to - * spawn, so it is settled. Culled workers are joined here too. + * The committer first: it is the only other thread that joins workers, + * so joining it settles 'joinable' before the loop reads it. No + * producer is left to spawn, and the committer already waits for every + * worker to deposit before it exits. */ + if (started) + pthread_join(pool->committer, NULL); for (i = 0; i < pool->nworkers; i++) { if (pool->workers[i].joinable) pthread_join(pool->workers[i].tid, NULL); } - if (started) - pthread_join(pool->committer, NULL); - if (pool->n_inline > 0 || pool->n_waited > 0 || n_culled > 0) - _nmsg_dprintf(2, "%s: %u of %u worker(s) at once, %u slot(s); %" PRIu64 " container(s) compressed by the reader, %" PRIu64 " wait(s) for a free slot, %" PRIu64 " worker(s) culled\n", __func__, npeak, - pool->nworkers, pool->depth, pool->n_inline, pool->n_waited, n_culled); + if (n_inline > 0 || n_waited > 0 || n_culled > 0) + _nmsg_dprintf(2, "%s: %u of %u worker(s) at once, %u slot(s); " + "%" PRIu64 " container(s) compressed by the reader, " + "%" PRIu64 " wait(s) for a free slot, " + "%" PRIu64 " worker(s) culled\n", + __func__, npeak, pool->nworkers, pool->depth, + n_inline, n_waited, n_culled); /* Read after the joins; the threads write it until they exit. */ res = pool->first_error; @@ -520,6 +495,7 @@ _output_async_destroy(nmsg_output_t output) pthread_mutex_lock(&ostr->c_lock); ostr->so_pool = NULL; ostr->so_pool_closing = false; + pthread_cond_broadcast(&ostr->c_drained); /* Any teardown behind us. */ pthread_mutex_unlock(&ostr->c_lock); for (i = 0; i < pool->nworkers; i++) @@ -528,6 +504,17 @@ _output_async_destroy(nmsg_output_t output) pthread_cond_destroy(&pool->commit_ready); pthread_mutex_destroy(&pool->lock); free(pool->workers); + + /* + * Every slot is empty by now: producers are gone, the threads are + * joined, and the committer writes every ticket that was issued. Swept + * anyway, so the invariant is checked rather than assumed. + */ + for (i = 0; i < pool->depth; i++) { + assert(pool->slots[i].state == slot_empty); + nmsg_container_destroy(&pool->slots[i].co); + free(pool->slots[i].buf); + } free(pool->slots); free(pool); @@ -566,8 +553,8 @@ async_all_written(const struct nmsg_ostr_async *pool) * the producer decides which worker runs and the rest go quiet. * * A worker idle for cull_secs gives up its place, down to min_workers. That - * decision and leaving the idle list are one lock hold, so a producer can never - * hand work to a thread on its way out. + * decision and clearing its idle flag are one lock hold, so a producer can + * never hand work to a thread on its way out. */ static void * async_worker(void *arg) @@ -586,6 +573,8 @@ async_worker(void *arg) nmsg_res res; if (slot == NULL) { + struct timespec deadline; + /* * Exit only once producers have stopped, so a * container queued just before shutdown is still @@ -601,8 +590,7 @@ async_worker(void *arg) timed_out = false; - if (!self->idle) - async_idle_push(pool, self); + self->idle = true; /* * No deadline with culling off, nor at the floor, @@ -611,14 +599,16 @@ async_worker(void *arg) * workers added on top are the ones that time out. */ if (pool->cull_secs == 0 || - pool->nlive <= pool->min_workers) { + pool->nlive <= pool->min_workers || + !async_cull_deadline(pool, &deadline)) { pthread_cond_wait(&self->ready, &pool->lock); } else { - struct timespec deadline; + int wait_res; - async_cull_deadline(pool, &deadline); - timed_out = pthread_cond_timedwait(&self->ready, - &pool->lock, &deadline) == ETIMEDOUT; + wait_res = pthread_cond_timedwait(&self->ready, + &pool->lock, + &deadline); + timed_out = wait_res == ETIMEDOUT; } continue; } @@ -640,8 +630,7 @@ async_worker(void *arg) pthread_cond_broadcast(&pool->commit_ready); } - if (self->idle) - async_idle_unlink(pool, self); + self->idle = false; self->exited = true; pool->nlive--; @@ -651,9 +640,40 @@ async_worker(void *arg) } /* - * The only thread that writes, and the only caller of - * _output_nmsg_frag_write(). Takes tickets strictly in order, so the file - * matches what the synchronous path would have produced. + * Join the workers culling has retired, freeing their records to be spawned + * into again. Runs on the committer because the join is unbounded -- the thread + * has returned but still has the C library's teardown to be scheduled for -- + * and the committer is the one thread that may block for it. + * + * Being the only reaper is what lets a producer skip a record on 'joinable' + * alone, so nothing can claim one while the lock is dropped here. + * Caller holds pool->lock. + */ +static void +async_reap_exited(struct nmsg_ostr_async *pool) +{ + unsigned i; + + for (i = 0; i < pool->nworkers; i++) { + struct async_worker *worker = &pool->workers[i]; + + if (!worker->exited) + continue; + + pthread_mutex_unlock(&pool->lock); + pthread_join(worker->tid, NULL); + pthread_mutex_lock(&pool->lock); + + /* Cleared last: until then the record is not free. */ + worker->exited = false; + worker->joinable = false; + } +} + +/* + * Takes tickets strictly in order, so the file matches what the synchronous + * path would have produced. The only writer while the pool is up: a producer + * writes inline only when there is no pool to take its ticket. */ static void * async_committer(void *arg) @@ -715,6 +735,8 @@ async_committer(void *arg) slot->state = slot_empty; pool->commit_next++; pthread_cond_broadcast(&pool->slot_free); + + async_reap_exited(pool); } out: @@ -750,67 +772,36 @@ async_start(struct nmsg_ostr_async *pool) pool->started = true; } -/* - * Claim a record for a new compressor, joining the culled thread that left it, - * or NULL when every record is running -- which is what holds the ceiling. - * - * The join runs with pool->lock dropped: a returned thread still has the C - * library's teardown to be scheduled for, and waiting under the lock would stop - * the committer writing. Caller holds pool->lock. - */ -static struct async_worker * -async_take_worker(struct nmsg_ostr_async *pool) -{ - struct async_worker *worker = NULL; - unsigned i; - - for (i = 0; i < pool->nworkers; i++) { - /* Free outright: never used, or already reaped. */ - if (!pool->workers[i].joinable) - return (&pool->workers[i]); - - if (worker == NULL && pool->workers[i].exited && - !pool->workers[i].reaping) - worker = &pool->workers[i]; - } - - if (worker == NULL) - return (NULL); - - worker->reaping = true; - pthread_mutex_unlock(&pool->lock); - pthread_join(worker->tid, NULL); - pthread_mutex_lock(&pool->lock); - - /* Cleared before the spawn: if it fails, there is nothing to join. */ - worker->joinable = false; - worker->exited = false; - worker->reaping = false; - - return (worker); -} - /* * Add a compressor, up to the ceiling. Called under pool->lock when a producer * finds no idle worker, so the pool grows to the load it sees. * - * Returns NULL if one could not be started: not fatal, the caller compresses - * that container itself. + * Takes only a free record: one a culled worker left stays off limits until the + * committer has joined it, so this never blocks the reader. Returns NULL if + * there is none or the spawn failed which is not fatal, the caller compresses that + * container itself. */ static struct async_worker * async_spawn_worker(struct nmsg_ostr_async *pool) { - struct async_worker *worker; + struct async_worker *worker = NULL; + unsigned i; int pthread_res; if (pool->spawn_failed) return (NULL); - worker = async_take_worker(pool); + for (i = 0; i < pool->nworkers; i++) { + if (!pool->workers[i].joinable) { + worker = &pool->workers[i]; + break; + } + } + if (worker == NULL) return (NULL); - assert(!worker->joinable && !worker->idle && worker->slot == NULL); + assert(!worker->idle && worker->slot == NULL); pthread_res = pthread_create(&worker->tid, NULL, async_worker, worker); if (pthread_res != 0) { @@ -833,14 +824,11 @@ async_spawn_worker(struct nmsg_ostr_async *pool) } /* - * Wait until every ticket issued so far has been written, and take any error - * the pool recorded. Under nmsg_io this cannot starve: check_close_event() - * holds io_output->refcount across the write, so no writer is inside - * nmsg_output_write() while a close runs. Callers driving nmsg_output_flush() - * from several threads have no such guarantee. + * Wait until every ticket below 'upto' has been written, and take any error the + * pool recorded. */ nmsg_res -_output_async_drain(struct nmsg_ostr_async *pool) +_output_async_drain(struct nmsg_ostr_async *pool, uint64_t upto) { nmsg_res res; @@ -848,7 +836,7 @@ _output_async_drain(struct nmsg_ostr_async *pool) return (nmsg_res_success); pthread_mutex_lock(&pool->lock); - while (!pool->failed && !async_all_written(pool)) + while (!pool->failed && pool->commit_next < upto) pthread_cond_wait(&pool->slot_free, &pool->lock); res = pool->first_error; pool->first_error = nmsg_res_success; @@ -870,26 +858,24 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, { struct nmsg_stream_output *ostr = output->stream; struct async_slot *slot; - struct async_worker *worker; + struct async_worker *worker = NULL; nmsg_res res = nmsg_res_success; uint8_t *buf; size_t buf_len; bool inline_compress = false; - bool committer_needed, pool_usable; pthread_mutex_lock(&pool->lock); - committer_needed = !pool->started && !pool->failed && !pool->shutdown; - if (committer_needed) - async_start(pool); - /* - * Read after the attempt, since async_start() sets started or failed. - * Only false when the committer could not start: teardown drains - * outstanding tickets before setting shutdown. + * No teardown can be running: it waits out the tickets already issued + * before it sets shutdown, and this producer is holding one. That is + * also what lets the slot wait below trust its slot is free. */ - pool_usable = pool->started && !pool->failed && !pool->shutdown; - if (!pool_usable) { + if (!pool->started && !pool->failed) + async_start(pool); + + /* Read after the attempt: async_start() sets started or failed. */ + if (!pool->started) { pthread_mutex_unlock(&pool->lock); _output_async_unref(ostr); return (false); @@ -903,7 +889,7 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, */ if (async_slot_busy(pool, ticket)) { pool->n_waited++; - while (async_slot_busy(pool, ticket) && !pool->shutdown) + while (async_slot_busy(pool, ticket)) pthread_cond_wait(&pool->slot_free, &pool->lock); } @@ -912,19 +898,21 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, if (ticket >= pool->issued) pool->issued = ticket + 1; + /* An idle compressor, or a new one if the pool may still grow. */ + if (!is_frag) { + worker = async_idle_take(pool); + if (worker == NULL) + worker = async_spawn_worker(pool); + } + if (is_frag) { /* Only the committer fragments; see async_committer(). */ slot->co = *co; *co = NULL; slot->state = slot_frag; pthread_cond_broadcast(&pool->commit_ready); - } else if ((worker = async_idle_pop(pool)) != NULL || - (worker = async_spawn_worker(pool)) != NULL) { - /* - * Hand the container over and get back to reading. The slot - * stays ours across the spawn's lock drop: no other ticket - * maps to it, and the committer waits until it is ready. - */ + } else if (worker != NULL) { + /* Hand the container over and get back to reading. */ slot->co = *co; *co = NULL; slot->state = slot_work; @@ -937,8 +925,11 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, inline_compress = true; } + /* + * Reported but not consumed: a write that never reached disk must still + * be there for the flush or close to find. Cleared by the drain. + */ res = pool->first_error; - pool->first_error = nmsg_res_success; pthread_mutex_unlock(&pool->lock); if (inline_compress) { diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index dfb80d3a..b1f480c6 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -33,7 +33,7 @@ _output_nmsg_flush(nmsg_output_t output) { nmsg_res res = nmsg_res_success; nmsg_res drain_res; nmsg_container_t old_c = NULL; - uint64_t ticket = 0; + uint64_t ticket = 0, upto; pthread_mutex_lock(&ostr->c_lock); @@ -55,6 +55,9 @@ _output_nmsg_flush(nmsg_output_t output) { nmsg_container_set_sequence(ostr->c, ostr->do_sequence); } + /* Everything sealed so far, including the container just taken. */ + upto = ostr->so_ticket; + pthread_mutex_unlock(&ostr->c_lock); /* @@ -70,7 +73,7 @@ _output_nmsg_flush(nmsg_output_t output) { } /* A flush means written, so wait out anything the pool still holds. */ - drain_res = _output_async_drain(pool); + drain_res = _output_async_drain(pool, upto); if (res == nmsg_res_success) res = drain_res; @@ -86,7 +89,7 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { struct nmsg_stream_output *ostr = output->stream; struct nmsg_ostr_async *pool; nmsg_container_t old_c, new_c; - nmsg_res res; + nmsg_res res, sub_res, pending = nmsg_res_success; uint64_t ticket; bool must_flush, is_buffered; @@ -161,13 +164,20 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { pthread_mutex_unlock(&ostr->c_lock); /* Release locked container to other threads. */ if (!must_flush) /* Nothing more to do here. */ - return (res); + return (pending != nmsg_res_success ? pending : res); /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ - res = container_submit(output, &old_c, false, ticket, pool); /* Write data from prior container. */ - if (res != nmsg_res_success) - return (res); + /* Write data from prior container. */ + sub_res = container_submit(output, &old_c, false, ticket, pool); + + /* + * Kept rather than returned: with a pool the result belongs to + * some earlier container, and returning here would drop this + * message. + */ + if (pending == nmsg_res_success) + pending = sub_res; /* Proceed to write current message to new container. */ goto retry; @@ -177,7 +187,7 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { res = container_submit(output, &old_c, true, ticket, pool); } - return (res); + return (pending != nmsg_res_success ? pending : res); } /* Private functions. */ diff --git a/nmsg/private.h b/nmsg/private.h index 7a5f3f13..37947776 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -344,13 +344,14 @@ struct nmsg_stream_output { bool do_sequence; atomic_uint_fast32_t so_sequence_num; uint64_t sequence_id; - uint64_t so_ticket; /* Next container ticket; c_lock. */ - unsigned so_inflight; /* Tickets owed to the pool; c_lock. */ - bool so_pool_closing; /* Pool teardown started; c_lock. */ - pthread_cond_t c_drained; /* so_inflight == 0; c_lock. */ - struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ - unsigned so_zmin; /* Compressors culling leaves; c_lock. */ - unsigned so_zcull; /* Idle seconds before a cull; c_lock. */ + /* The five below are guarded by c_lock. */ + uint64_t so_ticket; /* Next container ticket. */ + unsigned so_inflight; /* Tickets owed to the pool. */ + bool so_pool_closing;/* Teardown started. */ + unsigned so_zmin; /* Compressors culling leaves. */ + unsigned so_zcull; /* Idle seconds before a cull. */ + pthread_cond_t c_drained; /* so_inflight == 0. */ + struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ }; /* nmsg_callback_output: used by nmsg_output */ @@ -617,7 +618,7 @@ nmsg_res _output_async_destroy(nmsg_output_t); void _output_async_set_cull(struct nmsg_ostr_async *, unsigned, unsigned); void _output_async_counts(struct nmsg_ostr_async *, unsigned *, unsigned *, uint64_t *); -nmsg_res _output_async_drain(struct nmsg_ostr_async *); +nmsg_res _output_async_drain(struct nmsg_ostr_async *, uint64_t); struct nmsg_ostr_async *_output_async_ref(struct nmsg_stream_output *); void _output_async_unref(struct nmsg_stream_output *); bool _output_async_submit(struct nmsg_ostr_async *, nmsg_output_t, diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 12891d6a..324accfa 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -20,9 +20,6 @@ #include #include #include -#ifdef __linux__ -#include -#endif /* __linux__ */ #include #include #include @@ -340,7 +337,7 @@ static argv_t args[] = { ARGV_INT, &ctx.zasync, "n", - "compress file output on n threads" }, + "compress file output on n threads (-1 auto)" }, { '\0', "zcull", ARGV_INT, @@ -387,7 +384,13 @@ int main(int argc, char **argv) { #endif /* HAVE_LIBZMQ */ ctx.statsmods_loaded = statsmod_vec_init(1); + + /* Without it, outputs made before --zasync resolves get no pool. */ ctx.initial_outputs = output_vec_init(1); + if (ctx.initial_outputs == NULL) { + fprintf(stderr, "%s: out of memory\n", argv_program); + exit(EXIT_FAILURE); + } /* initialize the nmsg_io engine */ ctx.io = nmsg_io_init(); @@ -460,38 +463,13 @@ usage(const char *msg) { exit(msg == NULL ? EXIT_SUCCESS : EXIT_FAILURE); } -/* - * Cores this process may actually run on. - */ -long -nmsgtool_ncpu(void) { - long ncpu = -1; -#ifdef __linux__ - cpu_set_t set; - - if (sched_getaffinity(0, sizeof(set), &set) == 0) - ncpu = CPU_COUNT(&set); -#endif /* __linux__ */ - - if (ncpu < 1) - ncpu = sysconf(_SC_NPROCESSORS_ONLN); - if (ncpu < 1) - ncpu = 1; - - return (ncpu); -} - /* * How many compressor threads an output should get. A negative --zasync means * auto: demand is about two workers per input, since a reader can saturate - * roughly one core compressing, and supply is the cores the readers leave. - * - * The budget is split across the file outputs, which each get their own pool - * and share these cores. One worker is still enough to take compression off - * the reader, so no output drops below that. + * roughly one core compressing, bounded by the cores the readers leave. */ static unsigned -zworkers_count(nmsgtool_ctx *c) { +zworkers_count(const nmsgtool_ctx *c) { long ncpu; int n, spare; @@ -500,19 +478,35 @@ zworkers_count(nmsgtool_ctx *c) { if (c->zasync > 0) return ((unsigned) c->zasync); - ncpu = nmsgtool_ncpu(); + ncpu = my_ncpu(); n = 2 * c->n_inputs; + /* + * Split across the file outputs, which each get their own pool and + * share these cores. Not under --mirror, where every output is handed + * the whole stream and so needs the whole budget. + */ + if (!c->mirror) + n /= c->n_file_outputs; + + /* Room for one even where the readers outnumber the cores. */ spare = (int) ncpu - c->n_inputs; + if (spare < 1) + spare = 1; if (n > spare) n = spare; + + /* + * Applied last, and per output: a single input can carry several cores' + * worth on its own, which is what this floor is measured against. + */ if (n < NMSGTOOL_ZWORKERS_MIN) n = NMSGTOOL_ZWORKERS_MIN; - n /= c->n_file_outputs; - if (n < 1) - n = 1; + /* Never past what an explicit --zasync would be allowed. */ + if (n > (int) ncpu) + n = (int) ncpu; return ((unsigned) n); } @@ -522,6 +516,23 @@ zworkers_count(nmsgtool_ctx *c) { * the end of process_args() because the input count is not final until then: a * channel alias (-C) expands to its sockets after the outputs are created. */ +/* + * Cull policy first: setting the ceiling builds the pool, which reads it. The + * pool is an optimisation, so a failure to start one is reported, not fatal. + */ +static void +apply_zlib_workers(nmsgtool_ctx *c, nmsg_output_t output) +{ + nmsg_res res; + + nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); + + res = nmsg_output_set_zlib_workers(output, c->zworkers_resolved); + if (res != nmsg_res_success) + fprintf(stderr, "%s: no compressor pool: %s\n", argv_program, + nmsg_res_lookup(res)); +} + void setup_nmsg_output_workers(nmsgtool_ctx *c) { size_t i; @@ -532,16 +543,33 @@ setup_nmsg_output_workers(nmsgtool_ctx *c) { for (i = 0; i < output_vec_size(c->initial_outputs); i++) { nmsg_output_t output = output_vec_data(c->initial_outputs)[i]; - nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); - nmsg_output_set_zlib_workers(output, c->zworkers_resolved); + apply_zlib_workers(c, output); } output_vec_destroy(&c->initial_outputs); } - if (c->zworkers_resolved > 0 && c->debug >= 2) - fprintf(stderr, "%s: compressing on up to %u thread(s) per output, " - "keeping %d idle for %d second(s)\n", argv_program, - c->zworkers_resolved, c->zmin, c->zcull); + if (c->zworkers_resolved == 0) { + /* Nothing to apply a cull policy to; say so rather than not. */ + if (c->zcull != NMSGTOOL_ZCULL_DEFAULT || + c->zmin != NMSGTOOL_ZMIN_DEFAULT) + fprintf(stderr, "%s: --zcull and --zmin need --zasync " + "and an nmsg file output; ignored\n", argv_program); + return; + } + + if (c->debug >= 2) { + char cull[64]; + + if (c->zcull > 0) + snprintf(cull, sizeof(cull), + "culling to %d after %d idle second(s)", + c->zmin, c->zcull); + else + snprintf(cull, sizeof(cull), "never culling"); + + fprintf(stderr, "%s: compressing on up to %u thread(s) per " + "output, %s\n", argv_program, c->zworkers_resolved, cull); + } } void @@ -549,9 +577,7 @@ setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_buffered(output, !(c->unbuffered)); nmsg_output_set_endline(output, c->endline_str); nmsg_output_set_zlibout(output, c->zlibout); - /* Before the ceiling: setting that builds the pool, which reads this. */ - nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); - nmsg_output_set_zlib_workers(output, c->zworkers_resolved); + apply_zlib_workers(c, output); nmsg_output_set_source(output, c->set_source); nmsg_output_set_operator(output, c->set_operator); nmsg_output_set_group(output, c->set_group); diff --git a/src/nmsgtool.h b/src/nmsgtool.h index bc86b448..95507cf0 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -39,6 +39,7 @@ #endif /* HAVE_LIBRDKAFKA */ #include "libmy/argv.h" +#include "libmy/my_cpu.h" #include "libmy/vector.h" union nmsgtool_sockaddr { @@ -165,7 +166,6 @@ void add_zsock_input(nmsgtool_ctx *, const char *); void add_zsock_output(nmsgtool_ctx *, const char *); void add_filter_module(nmsgtool_ctx *, const char *); void add_stats_module(nmsgtool_ctx *, const char *); -long nmsgtool_ncpu(void); void pidfile_write(FILE *); void process_args(nmsgtool_ctx *); void setup_nmsg_input(nmsgtool_ctx *, nmsg_input_t); diff --git a/src/process_args.c b/src/process_args.c index 08b5ec8f..61a89add 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -125,21 +125,27 @@ process_args(nmsgtool_ctx *c) { if (c->mtu == 0) c->mtu = NMSG_WBUFSZ_JUMBO; - if (c->zasync < -1 || c->zasync > NMSGTOOL_ZWORKERS_MAX(nmsgtool_ncpu())) + if (c->zasync < -1 || c->zasync > NMSGTOOL_ZWORKERS_MAX(my_ncpu())) usage("--zasync must be -1 (choose), 0 (off), " "or a thread count no greater than the available cores"); + /* A container per message; a pool would spend a thread on each. */ + if (c->zasync != 0 && c->unbuffered) + usage("--zasync cannot be used with --unbuffered"); + if (c->zcull < 0) usage("--zcull must be 0 (never cull) or a number of seconds"); + if (c->zmin < 0) + usage("--zmin must be 0 or a number of compressor threads"); + /* - * Only checked for sanity here: the ceiling --zmin is a floor under is - * not settled until setup_nmsg_output_workers(), and libnmsg lowers it - * again to what the pool can run. -dd reports what it ended up as. + * Only checked against an explicit ceiling. Under --zasync -1 the + * ceiling is not settled until setup_nmsg_output_workers(), and libnmsg + * lowers the floor again to what the pool can run, reporting it at -dd. */ - if (c->zmin < 0 || c->zmin > NMSGTOOL_ZWORKERS_MAX(nmsgtool_ncpu())) - usage("--zmin must be 0 or a thread count no greater than " - "the available cores"); + if (c->zasync > 0 && c->zmin > c->zasync) + usage("--zmin cannot exceed --zasync"); if (c->vname == NULL && c->mname != NULL) c->vname = "base"; diff --git a/tests/test-zpool-cull.c b/tests/test-zpool-cull.c index 04d19492..265b90ba 100644 --- a/tests/test-zpool-cull.c +++ b/tests/test-zpool-cull.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -41,9 +42,25 @@ /* Long enough that a CULL_SECS deadline has certainly passed. */ #define SETTLE_SECS 3 +/* Payloads per burst. One container per burst is what the counts below rely on. */ +#define BURST 100 + +#if BURST * 64 > BUFSZ +#error "BURST no longer fits one container" +#endif + +/* Polling for a cull that has to happen, rather than guessing how long it takes. */ +#define POLL_STEP_MS 50 +#define POLL_MAX_MS 30000 + static nmsg_msgmod_t mod; -/* automake has no per-test timeout, so a wedged pool would hang forever. */ +/* + * automake has no per-test timeout, so a wedged pool would hang forever. Set + * well above the runtime: under valgrind or a loaded builder this test is + * slower by an order of magnitude, and a watchdog that fires then is + * indistinguishable from the deadlock it is meant to catch. + */ static void on_alarm(int sig __attribute__((unused))) { @@ -55,6 +72,33 @@ on_alarm(int sig __attribute__((unused))) _exit(1); } +/* nanosleep(), not sleep(): mixing sleep() with alarm() is unspecified. */ +static void +nap_ms(unsigned ms) +{ + struct timespec ts; + + ts.tv_sec = ms / 1000; + ts.tv_nsec = (long) (ms % 1000) * 1000000; + + while (nanosleep(&ts, &ts) != 0 && errno == EINTR) + ; +} + +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[1]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + static void fail(const char *what) { @@ -74,13 +118,16 @@ make_message(unsigned i) { char payload[48]; nmsg_message_t msg; - size_t len; + int len; msg = nmsg_message_init(mod); if (msg == NULL) fail("nmsg_message_init() failed"); len = snprintf(payload, sizeof(payload), "payload %u", i); + if (len < 0 || (size_t) len >= sizeof(payload)) + fail("snprintf() failed"); + if (nmsg_message_set_field(msg, "payload", 0, (const uint8_t *) payload, len) != nmsg_res_success) fail("nmsg_message_set_field() failed"); @@ -104,13 +151,19 @@ count_payloads(const char *path) if (input == NULL) fail("nmsg_input_open_file() failed"); - while (nmsg_input_read(input, &msg) == nmsg_res_success) { + for (;;) { + nmsg_res res = nmsg_input_read(input, &msg); + + if (res == nmsg_res_eof) + break; + if (res != nmsg_res_success) + fail("nmsg_input_read() failed"); + nmsg_message_destroy(&msg); n += 1; } - nmsg_input_close(&input); - close(fd); + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ return (n); } @@ -166,10 +219,28 @@ counts(nmsg_output_t output, unsigned *live, unsigned *peak, uint64_t *culled) _output_async_counts(pool, live, peak, culled); } +/* Wait for the pool to reach 'want' live compressors, or give up and say so. */ +static void +wait_for_live(nmsg_output_t output, unsigned want) +{ + unsigned live, waited; + + for (waited = 0; waited < POLL_MAX_MS; waited += POLL_STEP_MS) { + counts(output, &live, NULL, NULL); + if (live == want) + return; + nap_ms(POLL_STEP_MS); + } + + counts(output, &live, NULL, NULL); + if (live != want) + fail_count("pool did not settle", want, live); +} + /* * A worker starts, goes quiet and gives its place back; the pool then grows - * again from empty. Reaching zero is what exercises reusing a culled worker's - * record, which means joining the thread that left it. + * again from empty. Reaching zero is what exercises the committer reaping a + * culled worker before its record can be spawned into again. */ static void test_cull_to_empty(const char *path) @@ -178,21 +249,18 @@ test_cull_to_empty(const char *path) unsigned live, peak; uint64_t culled; - write_burst(output, 100, 0); + write_burst(output, BURST, 0); counts(output, &live, &peak, &culled); if (live != 1) fail_count("worker not started", 1, live); - sleep(SETTLE_SECS); + wait_for_live(output, 0); counts(output, &live, &peak, &culled); - if (live != 0) - fail_count("pool did not empty", 0, live); if (culled != 1) fail_count("culls recorded", 1, (unsigned) culled); - /* Growing again has to reuse the record the culled worker left. */ - write_burst(output, 100, 100); + write_burst(output, BURST, BURST); counts(output, &live, &peak, &culled); if (live != 1) fail_count("pool did not grow again", 1, live); @@ -200,8 +268,8 @@ test_cull_to_empty(const char *path) if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); - if (count_payloads(path) != 200) - fail_count("payloads written", 200, count_payloads(path)); + if (count_payloads(path) != 2 * BURST) + fail_count("payloads written", 2 * BURST, count_payloads(path)); } /* The floor is left alone, however long the pool stays quiet. */ @@ -212,8 +280,8 @@ test_floor(const char *path) unsigned live; uint64_t culled; - write_burst(output, 100, 0); - sleep(SETTLE_SECS); + write_burst(output, BURST, 0); + nap_ms(SETTLE_SECS * 1000); counts(output, &live, NULL, &culled); if (live != 1) @@ -233,12 +301,12 @@ test_cull_disabled(const char *path) unsigned live, peak; uint64_t culled; - write_burst(output, 100, 0); + write_burst(output, BURST, 0); counts(output, &live, &peak, &culled); if (live != peak) fail_count("workers lost before settling", peak, live); - sleep(SETTLE_SECS); + nap_ms(SETTLE_SECS * 1000); counts(output, &live, &peak, &culled); if (live != peak) @@ -260,8 +328,8 @@ test_floor_above_ceiling(const char *path) nmsg_output_t output = open_output(path, 1, 8, CULL_SECS); unsigned live; - write_burst(output, 100, 0); - sleep(SETTLE_SECS); + write_burst(output, BURST, 0); + nap_ms(SETTLE_SECS * 1000); counts(output, &live, NULL, NULL); if (live != 1) @@ -282,8 +350,8 @@ test_traffic_across_culls(const char *path) unsigned round, live; for (round = 0; round < 3; round++) { - write_burst(output, 100, round * 100); - sleep(SETTLE_SECS); + write_burst(output, BURST, round * BURST); + wait_for_live(output, 0); } counts(output, &live, NULL, NULL); @@ -293,16 +361,20 @@ test_traffic_across_culls(const char *path) if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); - if (count_payloads(path) != 300) - fail_count("payloads written", 300, count_payloads(path)); + if (count_payloads(path) != 3 * BURST) + fail_count("payloads written", 3 * BURST, count_payloads(path)); } -int main(void) { - char path[] = "/tmp/nmsg-zpool-cull.XXXXXX"; +int +main(void) +{ + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char path[] = "/tmp/nmsg-zpool-cull.XXXXXX"; int fd; - signal(SIGALRM, on_alarm); - alarm(120); + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); if (nmsg_init() != nmsg_res_success) fail("nmsg_init() failed"); @@ -316,6 +388,8 @@ int main(void) { if (fd < 0) fail("mkstemp() failed"); close(fd); + tmp_paths[0] = path; + atexit(unlink_tmp); test_cull_to_empty(path); test_floor(path); @@ -323,7 +397,5 @@ int main(void) { test_floor_above_ceiling(path); test_traffic_across_culls(path); - unlink(path); - return (0); } diff --git a/tests/test-zpool-mt.c b/tests/test-zpool-mt.c index 567b5bc9..8a661354 100644 --- a/tests/test-zpool-mt.c +++ b/tests/test-zpool-mt.c @@ -15,7 +15,7 @@ */ /* - * With several threads writing one output, the pool guarantees + * With several threads writing one output, the pool guarantees * containers reach the file in the order they were * sealed. A thread's own payloads must therefore appear in the file in the * order that thread wrote them. @@ -62,6 +62,20 @@ on_alarm(int sig __attribute__((unused))) _exit(1); } +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[1]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + static void fail(const char *what) { @@ -78,13 +92,16 @@ writer_thread(void *arg) for (i = 0; i < PER_THREAD; i++) { char payload[48]; nmsg_message_t msg; - size_t len; + int len; msg = nmsg_message_init(mod); if (msg == NULL) fail("nmsg_message_init() failed"); len = snprintf(payload, sizeof(payload), "%u:%u", w->id, i); + if (len < 0 || (size_t) len >= sizeof(payload)) + fail("snprintf() failed"); + if (nmsg_message_set_field(msg, "payload", 0, (const uint8_t *) payload, len) != nmsg_res_success) @@ -158,8 +175,7 @@ verify_order(const char *path) total += 1; } - nmsg_input_close(&input); - close(fd); + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ return (total); } @@ -191,7 +207,8 @@ run(const char *path, unsigned workers) } for (i = 0; i < NUM_THREADS; i++) - pthread_join(writers[i].thr, NULL); + if (pthread_join(writers[i].thr, NULL) != 0) + fail("pthread_join() failed"); if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); @@ -204,12 +221,16 @@ run(const char *path, unsigned workers) } } -int main(void) { - char path[] = "/tmp/nmsg-zpool-mt.XXXXXX"; +int +main(void) +{ + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char path[] = "/tmp/nmsg-zpool-mt.XXXXXX"; int fd; - signal(SIGALRM, on_alarm); - alarm(60); + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); if (nmsg_init() != nmsg_res_success) fail("nmsg_init() failed"); @@ -222,11 +243,11 @@ int main(void) { if (fd < 0) fail("mkstemp() failed"); close(fd); + tmp_paths[0] = path; + atexit(unlink_tmp); run(path, 1); run(path, 4); - unlink(path); - return (0); } \ No newline at end of file diff --git a/tests/test-zpool-order.c b/tests/test-zpool-order.c index 6b21de44..249e0383 100644 --- a/tests/test-zpool-order.c +++ b/tests/test-zpool-order.c @@ -34,6 +34,7 @@ #include #include "nmsg.h" +#include "private.h" #define NUM_PAYLOADS 4000 #define BUFSZ NMSG_WBUFSZ_JUMBO @@ -61,6 +62,20 @@ on_alarm(int sig __attribute__((unused))) _exit(1); } +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[2]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + static void fail(const char *what) { @@ -81,14 +96,16 @@ make_message(unsigned i) nmsg_message_t msg; struct timespec ts; size_t len; + int written; msg = nmsg_message_init(mod); if (msg == NULL) fail("nmsg_message_init() failed"); - len = snprintf(payload, sizeof(payload), "payload %u", i); - if (len >= BUFSZ / 2) - fail("payload too large; would fragment"); + written = snprintf(payload, sizeof(payload), "payload %u", i); + if (written < 0 || (size_t) written >= sizeof(payload)) + fail("snprintf() failed"); + len = (size_t) written; if (nmsg_message_set_field(msg, "payload", 0, (const uint8_t *) payload, len) != nmsg_res_success) @@ -119,13 +136,19 @@ count_payloads(const char *path) if (input == NULL) fail("nmsg_input_open_file() failed"); - while (nmsg_input_read(input, &msg) == nmsg_res_success) { + for (;;) { + nmsg_res res = nmsg_input_read(input, &msg); + + if (res == nmsg_res_eof) + break; + if (res != nmsg_res_success) + fail("nmsg_input_read() failed"); + nmsg_message_destroy(&msg); n += 1; } - nmsg_input_close(&input); - close(fd); + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ return (n); } @@ -195,6 +218,21 @@ write_corpus(const char *path, unsigned workers, bool late_enable, nmsg_rate_t r } } + /* + * Read before the close, which takes the pool down. Without this every + * assertion below would still hold if the pool had silently failed to + * start and everything had been compressed inline. + */ + if (workers > 0) { + unsigned peak = 0; + + if (output->stream->so_pool == NULL) + fail("output has no compressor pool"); + _output_async_counts(output->stream->so_pool, NULL, &peak, NULL); + if (peak == 0) + fail("no compressor thread ever ran"); + } + if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); } @@ -226,20 +264,24 @@ compare(const char *ref, const char *path, const char *what) fclose(fb); } -int main(void) { +int +main(void) +{ /* * 16 asks for a bigger reorder buffer than the rest; libnmsg clamps it * to what the machine allows, which exercises the clamp either way. */ static const unsigned counts[] = { 1, 4, 8, 16 }; - char ref[] = "/tmp/nmsg-zpool-ref.XXXXXX"; - char out[] = "/tmp/nmsg-zpool-out.XXXXXX"; + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char ref[] = "/tmp/nmsg-zpool-ref.XXXXXX"; + static char out[] = "/tmp/nmsg-zpool-out.XXXXXX"; nmsg_rate_t rate; unsigned i; int fd; - signal(SIGALRM, on_alarm); - alarm(30); + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); if (nmsg_init() != nmsg_res_success) fail("nmsg_init() failed"); @@ -253,10 +295,15 @@ int main(void) { if (fd < 0) fail("mkstemp() failed"); close(fd); + tmp_paths[0] = ref; + fd = mkstemp(out); if (fd < 0) fail("mkstemp() failed"); close(fd); + tmp_paths[1] = out; + + atexit(unlink_tmp); /* No pool: the reference every other run has to match. */ write_corpus(ref, 0, false, NULL); @@ -287,8 +334,5 @@ int main(void) { compare(ref, out, "rate-limited output"); nmsg_rate_destroy(&rate); - unlink(ref); - unlink(out); - return (0); } \ No newline at end of file From c5076cc31260a15e19c14ee020ff6da7ebc2eb55 Mon Sep 17 00:00:00 2001 From: Maximilian Terenzi Date: Fri, 21 Aug 2026 15:57:28 -0400 Subject: [PATCH 18/18] Audit: fix self-cull with culling off, a stuck flush, and --zcull/--zmin parsing --- Makefile.am | 6 ++- doc/docbook/nmsgtool.docbook | 95 ++++++++++++++---------------------- nmsg/output.h | 8 +-- nmsg/output_async.c | 70 +++++++++++++------------- nmsg/output_nmsg.c | 4 +- src/nmsgtool.c | 42 ++++++++-------- src/nmsgtool.h | 5 +- src/process_args.c | 27 ++++++++-- tests/test-zpool-cull.c | 37 +++++++++----- tests/test-zpool-mt.c | 17 ++++++- 10 files changed, 171 insertions(+), 140 deletions(-) diff --git a/Makefile.am b/Makefile.am index aa62d1d4..90468549 100644 --- a/Makefile.am +++ b/Makefile.am @@ -579,8 +579,12 @@ tests_test_zpool_order_SOURCES = tests/test-zpool-order.c TESTS += tests/test-zpool-mt check_PROGRAMS += tests/test-zpool-mt +tests_test_zpool_mt_LDFLAGS = -rdynamic +tests_test_zpool_mt_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) tests_test_zpool_mt_SOURCES = tests/test-zpool-mt.c -tests_test_zpool_mt_LDADD = nmsg/libnmsg.la TESTS += tests/test-zpool-cull check_PROGRAMS += tests/test-zpool-cull diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 16163bf5..13bf3963 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -664,54 +664,36 @@ the thread that filled them. A thread that is compressing is not reading its socket, and on a high volume channel that pause is long enough for the kernel receive queue to - overflow; handing the container to a compressor lets the - reader carry straight on. + overflow. Off by default. n is a ceiling rather than a thread count: compressors start only as load calls for them and are given back once idle, so an output that never saturates never pays for them. See and . - Compression is CPU-bound, so a thread per core is as far as - it can help; values above the cores available to the process - are rejected. + Compression is CPU-bound, so values above the cores available + to the process are rejected. A value of chooses the ceiling: two - per input, split across the file outputs, which each get - their own pool and share the same cores, then bounded by the - cores the readers leave spare. An output gets at least four - regardless, since a single input can carry several cores' - worth on its own, and never more than the core count. Sizing - it exactly is not important: a reader that finds every - compressor busy compresses the container itself, so too small - a pool costs nothing beyond the pool being idle. - - The cores counted are those in the process affinity - mask, which does not reflect a cgroup CPU quota; under a - container runtime that caps CPU rather than pinning it, pass - n explicitly. - - This is worth using when a single input carries more - data than one core can compress, and on file-to-file work - such as recompressing a capture. A channel spread over a - range of ports already compresses on every reader thread and - gains little; it also costs a little output size there, - because readers that never pause interleave their sources - more finely within a container, which compresses slightly - worse. With a single input the output is byte for byte - identical to compressing inline, whatever - n is. - - Compression itself still needs ; - without it a compressor only takes the serializing off the - reader. A pool holds a serialized container per queued - ticket, so it can add tens of megabytes of buffering to an - output. - - Containers are always written in the order they were - filled. Applies to file () outputs only, - and cannot be combined with - . + per input, divided across the file outputs but not under + , bounded by the cores the readers + leave spare, then never fewer than four nor more than the + core count. The cores counted are those in the process + affinity mask, which does not reflect a cgroup CPU quota. + Sizing it exactly does not matter, since a reader that finds + every compressor busy compresses the container itself. It + helps most where one input outruns a single core; a channel + spread over many ports already compresses on every reader + thread. + + Compression itself still needs , and + a pool holds a serialized container per queued ticket, so it + can add tens of megabytes of buffering. Containers are always + written in the order they were filled; with a single input + the output is byte for byte identical to compressing inline, + whatever n is. File + () outputs only, and cannot be combined + with . @@ -725,15 +707,13 @@ Containers always go to the lowest-numbered free compressor, so a pool that grew for a burst keeps the first - few busy and lets the rest fall quiet. The pool grows again - on demand, exactly as it did the first time, and a - compressor only ever leaves when it is holding nothing, so - the output is unaffected either way. - - This matters mainly for a long-running output that is - not being rotated: an output closed by - or gives its whole pool back at every - close regardless. + few busy and lets the rest fall quiet. A compressor only ever + leaves when it is holding nothing, and the pool grows again + on demand, so the output is unaffected either way. This + matters mainly for a long-running output that is not being + rotated: one closed by or + gives its whole pool back at every close + regardless. @@ -745,16 +725,13 @@ default. A value of lets the pool empty completely. - This is a floor on what culling may take away, not a - number of threads to start: compressors are still started - only on demand, so an output that has never been busy is - running none of them whatever this is set to. A value above - an explicit is rejected; under - it is lowered to the ceiling - chosen, which is reported at . - - One thread per file output is not culled in any case: - the thread that does the writing is not a compressor. + A floor on what culling may take away, not a number of + threads to start: compressors are still started only on + demand, so an output that has never been busy is running none + of them whatever this is set to. A value above an explicit + is rejected; under it is lowered to the ceiling chosen, which is + reported at . diff --git a/nmsg/output.h b/nmsg/output.h index 18e0a4af..c7ebccf8 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -401,10 +401,12 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); * \param[in] output nmsg_output_t object. * * \param[in] workers Maximum number of compressor threads, or 0 to compress - * inline (the default). + * inline (the default). Capped at the cores available to the process. * - * \return #nmsg_res_success, or #nmsg_res_failure on an unbuffered output. - * Setting 0 returns any write error the pool had yet to report. + * \return #nmsg_res_success, or #nmsg_res_failure on an unbuffered output; + * #nmsg_res_memfail or #nmsg_res_failure if the pool cannot be built, and + * success without a pool on an output this does not apply to. Setting 0 + * returns any write error the pool had yet to report. */ nmsg_res nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); diff --git a/nmsg/output_async.c b/nmsg/output_async.c index 8564f16b..1ccc9a42 100644 --- a/nmsg/output_async.c +++ b/nmsg/output_async.c @@ -43,9 +43,9 @@ */ /* - * Slots kept free for producers to deposit inline-compressed containers into - * while every worker is busy. Without it a saturated pool would have nowhere - * left to put anything. + * Slots beyond one per worker, so a producer can run ahead of the compressors + * instead of waiting on its own ticket's slot. Headroom, not a requirement: + * depth == nworkers would still make progress. */ #define ASYNC_REORDER_MARGIN 8 @@ -365,8 +365,6 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) pool->depth = depth; pool->nworkers = nworkers; pool->cull_clock = cull_clock; - pool->min_workers = async_min_workers_for(nworkers, zmin); - pool->cull_secs = zcull; pool->output = output; /* A no-op under c_lock if there is nothing to replace. */ @@ -374,17 +372,12 @@ _output_async_init(nmsg_output_t output, unsigned nworkers) pthread_mutex_lock(&ostr->c_lock); + pool->min_workers = async_min_workers_for(nworkers, ostr->so_zmin); + pool->cull_secs = ostr->so_zcull; + /* * Tickets span the stream, not the pool, so a pool built mid-stream - * must start where the stream got to. Seeded and published in one - * c_lock hold, so a ticket either predates the pool or is at or above - * commit_next, never below, where the committer would wait for it - * forever. - * - * That settles the pool's own liveness, not the byte order: a container - * from a ticket just before this may still be waiting to go out inline, - * and nothing holds this pool back for it. See - * nmsg_output_set_zlib_workers(). + * must start where the stream got to. */ pool->commit_next = pool->issued = ostr->so_ticket; @@ -583,7 +576,8 @@ async_worker(void *arg) if (pool->shutdown) break; - if (timed_out && pool->nlive > pool->min_workers) { + if (timed_out && pool->cull_secs > 0 && + pool->nlive > pool->min_workers) { pool->n_culled++; break; } @@ -598,8 +592,10 @@ async_worker(void *arg) * tied to particular threads: grow again and the * workers added on top are the ones that time out. */ - if (pool->cull_secs == 0 || - pool->nlive <= pool->min_workers || + bool park_indefinitely = pool->cull_secs == 0 || + pool->nlive <= pool->min_workers; + + if (park_indefinitely || !async_cull_deadline(pool, &deadline)) { pthread_cond_wait(&self->ready, &pool->lock); } else { @@ -761,9 +757,11 @@ async_start(struct nmsg_ostr_async *pool) /* * Nothing can be written without a committer, so abandon the * pool and compress inline. Nothing has been deposited yet to - * unwind. + * unwind, but a flush may already be waiting on a ticket that + * will now never be committed. */ pool->failed = true; + pthread_cond_broadcast(&pool->slot_free); _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, strerror(pthread_res)); return; @@ -898,31 +896,31 @@ bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, if (ticket >= pool->issued) pool->issued = ticket + 1; - /* An idle compressor, or a new one if the pool may still grow. */ - if (!is_frag) { - worker = async_idle_take(pool); - if (worker == NULL) - worker = async_spawn_worker(pool); - } - if (is_frag) { /* Only the committer fragments; see async_committer(). */ slot->co = *co; *co = NULL; slot->state = slot_frag; pthread_cond_broadcast(&pool->commit_ready); - } else if (worker != NULL) { - /* Hand the container over and get back to reading. */ - slot->co = *co; - *co = NULL; - slot->state = slot_work; - worker->slot = slot; - pthread_cond_signal(&worker->ready); } else { - /* Everyone busy and at the ceiling: compress here. */ - slot->state = slot_taken; - pool->n_inline++; - inline_compress = true; + /* An idle compressor, or a new one if the pool may still grow. */ + worker = async_idle_take(pool); + if (worker == NULL) + worker = async_spawn_worker(pool); + + if (worker != NULL) { + /* Hand the container over and get back to reading. */ + slot->co = *co; + *co = NULL; + slot->state = slot_work; + worker->slot = slot; + pthread_cond_signal(&worker->ready); + } else { + /* Everyone busy and at the ceiling: compress here. */ + slot->state = slot_taken; + pool->n_inline++; + inline_compress = true; + } } /* diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index b1f480c6..2d911838 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -164,7 +164,7 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { pthread_mutex_unlock(&ostr->c_lock); /* Release locked container to other threads. */ if (!must_flush) /* Nothing more to do here. */ - return (pending != nmsg_res_success ? pending : res); + goto out; /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ @@ -187,6 +187,8 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { res = container_submit(output, &old_c, true, ticket, pool); } +out: + /* An earlier container's error outranks this one's disposition. */ return (pending != nmsg_res_success ? pending : res); } diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 324accfa..0beefd29 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -34,8 +34,8 @@ /* Globals. */ /* - * Defaults set here rather than after argv_process(): 0 is a meaningful value - * for both cull options, so neither can use it as an "unset" marker. + * Cull defaults. 0 is meaningful for both, so the options are read as strings + * and these stand until process_args() sees one given. */ static nmsgtool_ctx ctx = { .zmin = NMSGTOOL_ZMIN_DEFAULT, @@ -340,14 +340,14 @@ static argv_t args[] = { "compress file output on n threads (-1 auto)" }, { '\0', "zcull", - ARGV_INT, - &ctx.zcull, + ARGV_CHAR_P, + &ctx.zcull_str, "secs", "drop a compressor idle after n secs (0 never)" }, { '\0', "zmin", - ARGV_INT, - &ctx.zmin, + ARGV_CHAR_P, + &ctx.zmin_str, "n", "never drop below n compressors" }, @@ -490,7 +490,7 @@ zworkers_count(const nmsgtool_ctx *c) { if (!c->mirror) n /= c->n_file_outputs; - /* Room for one even where the readers outnumber the cores. */ + /* What the readers leave. An upper bound, but see the floor below. */ spare = (int) ncpu - c->n_inputs; if (spare < 1) spare = 1; @@ -498,8 +498,10 @@ zworkers_count(const nmsgtool_ctx *c) { n = spare; /* - * Applied last, and per output: a single input can carry several cores' - * worth on its own, which is what this floor is measured against. + * Applied last, and per output, so it overrides the bound above: a + * single input can carry several cores' worth on its own, which is + * what this floor is measured against. Oversubscribing costs nothing + * on an output that never saturates, since workers start on demand. */ if (n < NMSGTOOL_ZWORKERS_MIN) n = NMSGTOOL_ZWORKERS_MIN; @@ -511,18 +513,12 @@ zworkers_count(const nmsgtool_ctx *c) { return ((unsigned) n); } -/* - * Resolve --zasync and apply it to the outputs that already exist. Deferred to - * the end of process_args() because the input count is not final until then: a - * channel alias (-C) expands to its sockets after the outputs are created. - */ /* * Cull policy first: setting the ceiling builds the pool, which reads it. The * pool is an optimisation, so a failure to start one is reported, not fatal. */ static void -apply_zlib_workers(nmsgtool_ctx *c, nmsg_output_t output) -{ +apply_zlib_workers(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_res res; nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); @@ -533,6 +529,11 @@ apply_zlib_workers(nmsgtool_ctx *c, nmsg_output_t output) nmsg_res_lookup(res)); } +/* + * Resolve --zasync and apply it to the outputs that already exist. Deferred to + * the end of process_args() because the input count is not final until then: a + * channel alias (-C) expands to its sockets after the outputs are created. + */ void setup_nmsg_output_workers(nmsgtool_ctx *c) { size_t i; @@ -549,9 +550,12 @@ setup_nmsg_output_workers(nmsgtool_ctx *c) { } if (c->zworkers_resolved == 0) { - /* Nothing to apply a cull policy to; say so rather than not. */ - if (c->zcull != NMSGTOOL_ZCULL_DEFAULT || - c->zmin != NMSGTOOL_ZMIN_DEFAULT) + /* Nothing to compress on; say so rather than not. */ + if (c->zasync != 0) + fprintf(stderr, "%s: --zasync needs an nmsg file " + "output; ignored\n", argv_program); + else if (c->zcull != NMSGTOOL_ZCULL_DEFAULT || + c->zmin != NMSGTOOL_ZMIN_DEFAULT) fprintf(stderr, "%s: --zcull and --zmin need --zasync " "and an nmsg file output; ignored\n", argv_program); return; diff --git a/src/nmsgtool.h b/src/nmsgtool.h index 95507cf0..26349879 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -62,8 +62,8 @@ VECTOR_GENERATE(output_vec, nmsg_output_t) /* * Ceiling for an explicit --zasync. Compression is CPU-bound, so a thread per * core is as far as it can help; workers start on demand, so this only bounds - * how far a saturated output may grow. libnmsg applies the same rule a margin - * tighter, since it sizes the reorder buffer that serves them. + * how far a saturated output may grow. libnmsg enforces the same ceiling, and + * sizes its reorder buffer a margin above it. */ #define NMSGTOOL_ZWORKERS_MAX(ncpu) (ncpu) @@ -81,6 +81,7 @@ typedef struct { argv_array_t r_pcapfile, r_pcapif; argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; + char *zmin_str, *zcull_str; int zasync; /* Compressor threads; -1 chooses. */ int zmin; /* Compressors culling leaves. */ int zcull; /* Idle seconds before a cull. */ diff --git a/src/process_args.c b/src/process_args.c index 61a89add..8c73a45d 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -63,6 +64,22 @@ droproot(nmsgtool_ctx *c, FILE *fp_pidfile) { argv_program, c->username); } +/* An integer in [min, max], or exit with 'what'. Rejects what atoi() would not. */ +static int +read_int_range(const char *str, int min, int max, const char *what) +{ + char *t; + long val; + + errno = 0; + val = strtol(str, &t, 0); + if (*str == '\0' || *t != '\0' || errno == ERANGE || + val < min || val > max) + usage(what); + + return (int) val; +} + /* Convert string to non-zero unsigned 32 bit val, returning zero on failure. */ static uint32_t read_uint32_nz(const char *str) @@ -133,11 +150,13 @@ process_args(nmsgtool_ctx *c) { if (c->zasync != 0 && c->unbuffered) usage("--zasync cannot be used with --unbuffered"); - if (c->zcull < 0) - usage("--zcull must be 0 (never cull) or a number of seconds"); + if (c->zcull_str != NULL) + c->zcull = read_int_range(c->zcull_str, 0, INT_MAX, + "--zcull must be 0 (never cull) or a number of seconds"); - if (c->zmin < 0) - usage("--zmin must be 0 or a number of compressor threads"); + if (c->zmin_str != NULL) + c->zmin = read_int_range(c->zmin_str, 0, INT_MAX, + "--zmin must be 0 or a number of compressor threads"); /* * Only checked against an explicit ceiling. Under --zasync -1 the diff --git a/tests/test-zpool-cull.c b/tests/test-zpool-cull.c index 265b90ba..f2f8ffb9 100644 --- a/tests/test-zpool-cull.c +++ b/tests/test-zpool-cull.c @@ -246,30 +246,34 @@ static void test_cull_to_empty(const char *path) { nmsg_output_t output = open_output(path, 4, 0, CULL_SECS); - unsigned live, peak; + unsigned live; uint64_t culled; + /* Polled, not read once: the worker arms its deadline as it finishes. */ write_burst(output, BURST, 0); - counts(output, &live, &peak, &culled); - if (live != 1) - fail_count("worker not started", 1, live); + wait_for_live(output, 1); wait_for_live(output, 0); - counts(output, &live, &peak, &culled); + counts(output, &live, NULL, &culled); if (culled != 1) fail_count("culls recorded", 1, (unsigned) culled); + /* + * Two bursts: a culled worker's record is not free again until the + * committer reaps it, which happens on the next commit. Where the + * ceiling is 1 -- a single-core builder -- that commit is the first + * burst's, so the pool cannot grow until the second. + */ write_burst(output, BURST, BURST); - counts(output, &live, &peak, &culled); - if (live != 1) - fail_count("pool did not grow again", 1, live); + write_burst(output, BURST, 2 * BURST); + wait_for_live(output, 1); if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); - if (count_payloads(path) != 2 * BURST) - fail_count("payloads written", 2 * BURST, count_payloads(path)); + if (count_payloads(path) != 3 * BURST) + fail_count("payloads written", 3 * BURST, count_payloads(path)); } /* The floor is left alone, however long the pool stays quiet. */ @@ -319,21 +323,28 @@ test_cull_disabled(const char *path) } /* - * A floor above the ceiling has nothing to cull, and must be lowered to it - * rather than quietly disable the policy. + * A floor above the ceiling leaves nothing to cull, and the pool has to keep + * working rather than wedge or cull anyway. + * + * Not a test of async_min_workers_for()'s clamp, which cannot be one: nlive + * never exceeds the ceiling, so `nlive > min_workers` is false whether the + * floor was lowered or left at 8. The clamp's only effect is its -dd line. */ static void test_floor_above_ceiling(const char *path) { nmsg_output_t output = open_output(path, 1, 8, CULL_SECS); unsigned live; + uint64_t culled; write_burst(output, BURST, 0); nap_ms(SETTLE_SECS * 1000); - counts(output, &live, NULL, NULL); + counts(output, &live, NULL, &culled); if (live != 1) fail_count("clamped floor not held", 1, live); + if (culled != 0) + fail_count("culls under a clamped floor", 0, (unsigned) culled); if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed"); diff --git a/tests/test-zpool-mt.c b/tests/test-zpool-mt.c index 8a661354..8fcc45d4 100644 --- a/tests/test-zpool-mt.c +++ b/tests/test-zpool-mt.c @@ -38,6 +38,7 @@ #include #include "nmsg.h" +#include "private.h" #define NUM_THREADS 4 #define PER_THREAD 5000 @@ -184,7 +185,7 @@ static void run(const char *path, unsigned workers) { struct writer writers[NUM_THREADS]; - unsigned i, total; + unsigned i, total, peak = 0; int fd; fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); @@ -197,7 +198,8 @@ run(const char *path, unsigned workers) nmsg_output_set_buffered(output, true); nmsg_output_set_zlibout(output, true); - nmsg_output_set_zlib_workers(output, workers); + if (nmsg_output_set_zlib_workers(output, workers) != nmsg_res_success) + fail("nmsg_output_set_zlib_workers() failed"); for (i = 0; i < NUM_THREADS; i++) { writers[i].id = i; @@ -210,6 +212,17 @@ run(const char *path, unsigned workers) if (pthread_join(writers[i].thr, NULL) != 0) fail("pthread_join() failed"); + /* + * Read before the close, which takes the pool down. Without it the + * ordering check below would still pass with no pool at all -- the + * reordering a pool prevents is a race that need not occur. + */ + if (output->stream->so_pool == NULL) + fail("output has no compressor pool"); + _output_async_counts(output->stream->so_pool, NULL, &peak, NULL); + if (peak == 0) + fail("no compressor thread ever ran"); + if (nmsg_output_close(&output) != nmsg_res_success) fail("nmsg_output_close() failed");