diff --git a/.gitignore b/.gitignore index 5c94a3dd7..edea6ec68 100644 --- a/.gitignore +++ b/.gitignore @@ -65,5 +65,10 @@ tests/test-io tests/test-misc tests/test-parse tests/test-private +tests/test-zpool-cull +tests/test-zpool-mt +tests/test-zpool-order +tests/testzmq.json +tests/testzmq.sock tests/*/*.out tests/*/test.sh diff --git a/Makefile.am b/Makefile.am index e12d51a01..904685494 100644 --- a/Makefile.am +++ b/Makefile.am @@ -216,6 +216,7 @@ LIBNMSG_LIB_MODULES = \ nmsg/msgmodset.c \ nmsg/nmsg.c \ nmsg/output.c \ + nmsg/output_async.c \ nmsg/output_json.c \ nmsg/output_nmsg.c \ nmsg/output_pres.c \ @@ -244,6 +245,7 @@ LIBNMSG_LIB_MODULES = \ nmsg_libnmsg_la_SOURCES = \ libmy/crc32c.h \ libmy/list.h \ + libmy/my_cpu.h \ libmy/my_time.h \ libmy/my_rate.h \ libmy/tree.h \ @@ -419,6 +421,7 @@ src_nmsgtool_SOURCES = \ libmy/argv.c \ libmy/argv.h \ libmy/argv_loc.h \ + libmy/my_cpu.h \ src/daemon.c \ src/getsock.c \ src/io.c \ @@ -436,6 +439,11 @@ src_nmsgtool_SOURCES = \ ## # +# Tests that reach private symbols link the objects: libnmsg.la exports +# nmsg_* only, and the dlopened base msgmod resolves its own from the +# executable, which is what -rdynamic is for. +PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o) + TESTS_ENVIRONMENT = NMSG_MSGMOD_DIR=$(abs_top_builddir)/nmsg/base/.libs TESTS_ENVIRONMENT += abs_top_builddir='$(abs_top_builddir)' abs_top_srcdir='$(abs_top_srcdir)' @@ -537,7 +545,6 @@ TESTS += tests/test-private check_PROGRAMS += tests/test-private tests_test_private_LDFLAGS = -rdynamic tests_test_private_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS) -PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o) tests_test_private_LDADD = \ $(PRIVATE_TEST_MODULES) \ nmsg/nmsg.pb-c.o \ @@ -561,6 +568,33 @@ check_PROGRAMS += tests/test-nmsg_output_set_rate tests_test_nmsg_output_set_rate_SOURCES = tests/test-nmsg_output_set_rate.c tests_test_nmsg_output_set_rate_LDADD = nmsg/libnmsg.la +TESTS += tests/test-zpool-order +check_PROGRAMS += tests/test-zpool-order +tests_test_zpool_order_LDFLAGS = -rdynamic +tests_test_zpool_order_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) +tests_test_zpool_order_SOURCES = tests/test-zpool-order.c + +TESTS += tests/test-zpool-mt +check_PROGRAMS += tests/test-zpool-mt +tests_test_zpool_mt_LDFLAGS = -rdynamic +tests_test_zpool_mt_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) +tests_test_zpool_mt_SOURCES = tests/test-zpool-mt.c + +TESTS += tests/test-zpool-cull +check_PROGRAMS += tests/test-zpool-cull +tests_test_zpool_cull_LDFLAGS = -rdynamic +tests_test_zpool_cull_LDADD = \ + $(PRIVATE_TEST_MODULES) \ + nmsg/nmsg.pb-c.o \ + $(LIBNMSG_LIB_DEPS) +tests_test_zpool_cull_SOURCES = tests/test-zpool-cull.c + DISTCLEANFILES += tests/group-operator-source-tests/test*.out DISTCLEANFILES += tests/nmsg-dns-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out diff --git a/README.md b/README.md index 215274036..31bb4ff81 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,6 @@ to the `configure` script. Support for `librdkafka` can be disabled by passing the `--without-librdkafka` parameter to the `configure` script. -Support for `json-c` can be disabled by passing the `--without-json-c` parameter -to the `configure` script. - The documentation for the `libnmsg` API is located in the `doc/doxygen/html` directory. To rebuild the API documentation, run `make html`. This requires Doxygen to be installed. diff --git a/configure.ac b/configure.ac index 950ea3228..8f2cca9ff 100644 --- a/configure.ac +++ b/configure.ac @@ -96,6 +96,9 @@ AC_CHECK_FUNCS([clock_gettime]) AC_SEARCH_LIBS([clock_nanosleep], [rt]) AC_CHECK_FUNCS([clock_nanosleep]) +AC_SEARCH_LIBS([pthread_condattr_setclock], [pthread]) +AC_CHECK_FUNCS([pthread_condattr_setclock]) + AC_SEARCH_LIBS([dlopen], [dl]) AC_CHECK_FUNCS([dlopen]) @@ -168,14 +171,7 @@ else use_libzmq="false" fi -AC_ARG_WITH([json-c], AS_HELP_STRING([--without-json-c], [Disable json-c support])) -if test "x$with_json_c" != "xno"; then - PKG_CHECK_MODULES([json_c], [json-c >= 0.13.0]) - AC_DEFINE([HAVE_JSON_C], [1], [Define to 1 if json-c support is enabled.]) - use_json_c="true" -else - use_json_c="false" -fi +PKG_CHECK_MODULES([json_c], [json-c >= 0.13.0]) AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ]) AC_CHECK_LIB([z], [deflate], [], [ AC_MSG_ERROR([required library not found]) ]) @@ -270,7 +266,6 @@ AC_MSG_RESULT([ bigendian: ${ac_cv_c_bigendian} libzmq support: ${use_libzmq} librdkafka support: ${use_librdkafka} - json-c support: ${use_json_c} building html docs: ${DOC_HTML_MSG} building manpage docs: ${DOC_MAN_MSG} diff --git a/debian/libnmsg8.symbols b/debian/libnmsg8.symbols index 7c4e91395..1a9515fdf 100644 --- a/debian/libnmsg8.symbols +++ b/debian/libnmsg8.symbols @@ -30,6 +30,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_input_open_json@Base 0.10.0 nmsg_input_open_kafka_endpoint@Base 1.2.0 nmsg_input_open_kafka_json@Base 1.2.0 + nmsg_input_open_kafka_payload@Base 1.4.0 nmsg_input_open_null@Base 0.7.0 nmsg_input_open_pcap@Base 0.5.0 nmsg_input_open_pres@Base 0.5.0 @@ -138,6 +139,7 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_open_sock@Base 0.5.0 nmsg_output_open_zmq@Base 0.14.0 nmsg_output_open_zmq_endpoint@Base 0.14.0 + nmsg_output_open_kafka_payload@Base 1.4.0 nmsg_output_set_buffered@Base 0.5.0 nmsg_output_set_endline@Base 0.5.0 nmsg_output_set_filter_msgtype@Base 0.5.0 @@ -146,6 +148,8 @@ libnmsg.so.8 libnmsg8 #MINVER# nmsg_output_set_operator@Base 0.5.0 nmsg_output_set_rate@Base 0.5.0 nmsg_output_set_source@Base 0.5.0 + nmsg_output_set_zlib_cull@Base 1.4.0 + nmsg_output_set_zlib_workers@Base 1.4.0 nmsg_output_set_zlibout@Base 0.5.0 nmsg_output_write@Base 0.11.1 nmsg_pcap_filter@Base 0.6.5 diff --git a/doc/docbook/nmsgtool.1 b/doc/docbook/nmsgtool.1 index 0b7bad9de..3d398c265 100644 --- a/doc/docbook/nmsgtool.1 +++ b/doc/docbook/nmsgtool.1 @@ -2,12 +2,12 @@ .\" Title: nmsgtool .\" Author: [FIXME: author] [see http://www.docbook.org/tdg5/en/html/author] .\" Generator: DocBook XSL Stylesheets v1.79.2 -.\" Date: 10/02/2024 +.\" Date: 04/22/2026 .\" Manual: .\" Source: .\" Language: English .\" -.TH "NMSGTOOL" "1" "10/02/2024" "" "" +.TH "NMSGTOOL" "1" "04/22/2026" "" "" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -118,6 +118,8 @@ Writing ASCII presentation form data to a file\&. See the option\&. .RE .PP +There are more input and output types supported, depending upon the installation\&. See below in the options list\&. +.PP Reading or writing data in a non\-NMSG format requires the use of an external module (called an "nmsgpb module") to convert to or from NMSG format\&. \fBnmsgtool\fR selects an nmsgpb module based on a vendor ID and message type\&. For input data, these fields must be set with the @@ -142,15 +144,15 @@ Increment debugging level\&. \fB\-dd\fR is verbose and \fB\-dddd\fR -is very verbose\&. -If the +is very verbose\&. If the \fBNMSG_KAFKA_LOG_LEVEL\fR -environment variable is set and a Kafka input/output is used, Kafka will log non-errors at the specified logging level\&. +environment variable is set and a Kafka input/output is used, Kafka will log non\-errors at the specified logging level\&. .RE .PP -\fB\-v\fR \fIversion\fR +\fB\-v\fR, \fB\-\-version\fR .RS 4 -Print the version number of \fBnmsgtool\fR. +Print +\fBnmsgtool\fR\*(Aqs version number\&. .RE .PP \fB\-V\fR \fIvendor\fR, \fB\-\-vendor\fR \fIvendor\fR @@ -167,24 +169,34 @@ Set the message type field of generated NMSG payloads to the message type identi .PP \fB\-B\fR \fIbyterate\fR, \fB\-\-byterate\fR \fIbyterate\fR .RS 4 -Set the maximum bytes-per-second that libnmsg will process from file inputs. +Set the maximum bytes\-per\-second that libnmsg will process from file inputs\&. .RE .PP -\fB\-D\fR \fIdaemon\fR +\fB\-D\fR, \fB\-\-daemon\fR .RS 4 -Fork \fBnmsgtool\fR into the background as a daemon process. +Fork +\fBnmsgtool\fR +into the background as a daemon process\&. .RE .PP \fB\-P\fR \fIpidfile\fR, \fB\-\-pidfile\fR \fIpidfile\fR .RS 4 -Write \fBnmsgtool\fR's process ID into a file identified by \fIpidfile\fR. When -\fBnmsgtool\fR exits, this file's contents will be erased. +Write +\fBnmsgtool\fR\*(Aqs process ID into a file identified by +\fIpidfile\fR\&. When +\fB nmsgtool\fR +exits, this file\*(Aqs contents will be erased\&. .RE .PP \fB\-U\fR \fIusername\fR, \fB\-\-username\fR \fIusername\fR .RS 4 -Attempt to drop root privileges and run \fBnmsgtool\fR as user \fIusername\fR. -If the current user is \fIusername\fR, \fBnmsgtool\fR will exit. +Attempt to drop root privileges and run +\fBnmsgtool \fR +as user +\fIusername\fR\&. If the current user is +\fIusername\fR, +\fB nmsgtool\fR +will exit\&. .RE .PP \fB\-e\fR \fIendline\fR, \fB\-\-endline\fR \fIendline\fR @@ -264,9 +276,48 @@ environment variable\&. Read NMSG payloads in either binary or JSON format from a Kafka endpoint\&. The address \fIkafka\fR has format -\fBproto:topic[#partition|%group_id]@broker[:port][,offset]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. Also optional is an offset consisting of either a numerical value or the string \*(Aqoldest\*(Aq or \*(Aqnewest\*(Aq in order to start retrieval at the oldest/newest messages in the Kafka topic\&. An example of a possible +\fBproto:topic[#partition|%group_id]@broker[:port][,offset]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. Also optional is an offset consisting of either a numerical value or the string \*(Aqoldest\*(Aq or \*(Aqnewest\*(Aq in order to start retrieval at the oldest/newest messages in the Kafka topic\&. +.sp +There are three \fIkafka\fR -endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&. Configuration for Kafka can be supplied through the +endpoint proto types defined: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +json: to read json payloads from each kafka record +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsg: to read a full binary NMSG container from each kafka record +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsgp: to read a single binary NMSG payload from each kafka record +.RE +.sp +Example: "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&. +.sp +Configuration for Kafka can be supplied through the \fBNMSG_KAFKA_CONFIG\fR environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&. .RE @@ -355,17 +406,6 @@ environment variable\&. Read NMSG payloads from a file\&. .RE .PP -\fB\-f\fR \fIfile\fR, \fB\-\-readpres\fR \fIfile\fR -.RS 4 -Read presentation format data from a file and convert to NMSG payloads\&. This option is dependent on the -\fB\-V\fR -and -\fB\-T\fR -options being set in order to select a specific nmsgpb module to perform presentation format to NMSG payload conversion\&. Not all nmsgpb modules necessarily support this conversion method, in which case -\fBnmsgtool\fR -will print a "function not implemented" message\&. -.RE -.PP \fB\-j\fR \fIfile\fR, \fB\-\-readjson\fR \fIfile\fR .RS 4 Read JSON format data from a file\&. See documentation for @@ -514,9 +554,50 @@ Write NMSG payloads to a file\&. Write NMSG payloads in either binary or JSON format to a Kafka endpoint\&. The address \fIkafka\fR has format -\fBproto:topic[#partition|%group_id]@broker[:port]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. An example of a possible +\fBproto:topic[#partition|%group_id]@broker[:port]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. +.sp +There are three \fIkafka\fR -endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. Note that nmsgtool ignores offsets for Kafka producers\&. Configuration for Kafka can be supplied through the +endpoint proto types defined: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +json: to write JSON payloads to Kafka\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsg: to write a full binary NMSG container in each kafka record\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +nmsgp: to write a single binary NMSG payload in each kafka record\&. +.RE +.sp +Example: "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. +.sp +Note that nmsgtool ignores offsets for Kafka producers\&. +.sp +Configuration for Kafka can be supplied through the \fBNMSG_KAFKA_CONFIG\fR environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&. .RE @@ -800,7 +881,7 @@ To read NMSG payloads from a file and write them to a ZeroMQ "PUSH" socket over .PP This attempts to connect to a TCP reader on 127\&.0\&.0\&.1:5555, such as the nmsgtool command in the previous example\&. .PP -To read NMSG payloads from an SIE channel named "ch222" and write them to stdout while writing IO stats every 3 seconds to a file name "ch222.stats": +To read NMSG payloads from an SIE channel named "ch222" and write them to stdout while writing IO stats every 3 seconds to a file name "ch222\&.stats": .sp .if n \{\ .RS 4 diff --git a/doc/docbook/nmsgtool.docbook b/doc/docbook/nmsgtool.docbook index 6e0e5901d..13bf3963c 100644 --- a/doc/docbook/nmsgtool.docbook +++ b/doc/docbook/nmsgtool.docbook @@ -76,6 +76,9 @@ + There are more input and output types supported, depending upon the installation. See below in the + options list. + Reading or writing data in a non-NMSG format requires the use of an external module (called an "nmsgpb module") to convert to or from NMSG format. nmsgtool selects an @@ -114,7 +117,7 @@ verbose and is very verbose. If the NMSG_KAFKA_LOG_LEVEL environment variable is set and a Kafka input/output is used, Kafka will - log non-errors at the specified logging level. + log non-errors at the specified logging level. @@ -125,7 +128,7 @@ Print nmsgtool's version number. - + vendor vendor @@ -153,7 +156,7 @@ byterate byterate - Set the maximum bytes-per-second that libnmsg will + Set the maximum bytes-per-second that libnmsg will process from file inputs. @@ -162,7 +165,7 @@ - Fork nmsgtool into the background as a daemon + Fork nmsgtool into the background as a daemon process. @@ -171,23 +174,23 @@ pidfile pidfile - Write nmsgtool's process ID into a file - identified by pidfile. When + Write nmsgtool's process ID into a file + identified by pidfile. When nmsgtool exits, this file's contents will be erased. - + username username Attempt to drop root privileges and run nmsgtool - as user username. If the + as user username. If the current user is username, nmsgtool will exit. - + endline endline @@ -293,13 +296,26 @@ Also optional is an offset consisting of either a numerical value or the string 'oldest' or 'newest' in order to start retrieval at the oldest/newest messages in the Kafka topic. - An example of a possible kafka endpoint is - "nmsg:ch202#0@kafka.example.com:9092,3000" to indicate that nmsgtool shall read nmsg - containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at - kafka.example.com, port 9092. - Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. - This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. + + There are three kafka endpoint proto types + defined: + + + + json: to read json payloads from each kafka record + + + nmsg: to read a full binary NMSG container from each kafka record + + + nmsgp: to read a single binary NMSG payload from each kafka record + + + Example: "nmsg:ch202#0@kafka.example.com:9092,3000" to indicate that nmsgtool shall read nmsg containers + from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka.example.com, port 9092. + Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. + This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. @@ -401,21 +417,6 @@ - - file - file - - Read presentation format data from a file and convert - to NMSG payloads. This option is dependent on the - and options being - set in order to select a specific nmsgpb module to perform - presentation format to NMSG payload conversion. Not all nmsgpb - modules necessarily support this conversion method, in which - case nmsgtool will print a "function not - implemented" message. - - - file file @@ -549,13 +550,27 @@ Write NMSG payloads in either binary or JSON format to a Kafka endpoint. The address kafka has format proto:topic[#partition|%group_id]@broker[:port]. Either a partition number or a consumer group ID may be optionally supplied. - An example of a possible kafka endpoint is - "nmsg:ch202#0@kafka.example.com:9092" to indicate that nmsgtool shall write - nmsg containers to topic "ch202" on partition 0 to Kafka - broker kafka.example.com, port 9092. - Note that nmsgtool ignores offsets for Kafka producers. - Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. - This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. + + + There are three kafka endpoint proto types + defined: + + + + json: to write JSON payloads to Kafka. + + + nmsg: to write a full binary NMSG container in each kafka record. + + + nmsgp: to write a single binary NMSG payload in each kafka record. + + + Example: "nmsg:ch202#0@kafka.example.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on + partition 0 to Kafka broker kafka.example.com, port 9092. + Note that nmsgtool ignores offsets for Kafka producers. + Configuration for Kafka can be supplied through the NMSG_KAFKA_CONFIG environment variable. + This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format. @@ -641,6 +656,85 @@ + + n + + Compress written NMSG containers on + n separate threads rather than on + the thread that filled them. A thread that is compressing is + not reading its socket, and on a high volume channel that + pause is long enough for the kernel receive queue to + overflow. + + Off by default. n is a + ceiling rather than a thread count: compressors start only as + load calls for them and are given back once idle, so an + output that never saturates never pays for them. See + and . + Compression is CPU-bound, so values above the cores available + to the process are rejected. + + A value of chooses the ceiling: two + per input, divided across the file outputs but not under + , bounded by the cores the readers + leave spare, then never fewer than four nor more than the + core count. The cores counted are those in the process + affinity mask, which does not reflect a cgroup CPU quota. + Sizing it exactly does not matter, since a reader that finds + every compressor busy compresses the container itself. It + helps most where one input outruns a single core; a channel + spread over many ports already compresses on every reader + thread. + + Compression itself still needs , and + a pool holds a serialized container per queued ticket, so it + can add tens of megabytes of buffering. Containers are always + written in the order they were filled; with a single input + the output is byte for byte identical to compressing inline, + whatever n is. File + () outputs only, and cannot be combined + with . + + + + + secs + + Give up a compressor thread that has been idle for + secs seconds, 300 by default. A + value of never gives one up, leaving a + pool at the largest it ever needed to be. + + Containers always go to the lowest-numbered free + compressor, so a pool that grew for a burst keeps the first + few busy and lets the rest fall quiet. A compressor only ever + leaves when it is holding nothing, and the pool grows again + on demand, so the output is unaffected either way. This + matters mainly for a long-running output that is not being + rotated: one closed by or + gives its whole pool back at every close + regardless. + + + + + n + + Never let take an output + below n compressor threads, 1 by + default. A value of lets the pool empty + completely. + + A floor on what culling may take away, not a number of + threads to start: compressors are still started only on + demand, so an output that has never been busy is running none + of them whatever this is set to. A value above an explicit + is rejected; under it is lowered to the ceiling chosen, which is + reported at . + + + diff --git a/libmy/b32_decode.c b/libmy/b32_decode.c deleted file mode 100644 index 53038114b..000000000 --- a/libmy/b32_decode.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2015 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * See RFC 4648 for details about Base 32 hex encoding: - * http://tools.ietf.org/html/rfc4648 - */ - -#include -#include - -#include "b32_decode.h" - -#ifndef G_N_ELEMENTS -#define G_N_ELEMENTS(arr) (sizeof (arr) / sizeof ((arr)[0])) -#endif - -#define ZERO(x) memset((x), 0, sizeof *(x)) - -static inline void * -ptr_add_offset(void *p, size_t offset) -{ - /* Using size_t instead of 'char *' because pointers don't wrap. */ - return (void *) ((size_t) p + offset); -} - -static inline const void * -cast_to_constpointer(const void *p) -{ - return p; -} - -static inline bool -is_ascii_lower(int c) -{ - return c >= 97 && c <= 122; /* a-z */ -} - -static inline int -ascii_toupper(int c) -{ - return is_ascii_lower(c) ? c - 32 : c; -} - -static inline size_t -ptr_diff(const void *a, const void *b) -{ - return (const char *) a - (const char *) b; -} - -static const char base32_alphabet[32] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V' -}; - -static char base32_map[(size_t) (unsigned char) -1 + 1]; - -/** - * Decode a base32 encoding of `len' bytes of `data' into the buffer `dst'. - * - * @param dst destination buffer - * @param size length of destination - * @param data start of data to decode - * @param len amount of encoded data to decode - * - * @return the amount of bytes decoded into the destination. - */ -size_t -base32_decode(void *dst, size_t size, const char *data, size_t len) -{ - const char *end = ptr_add_offset(dst, size); - const unsigned char *p = cast_to_constpointer(data); - char s[8]; - char *q = dst; - int pad = 0; - size_t i, si; - - if (0 == base32_map[0]) { - for (i = 0; i < G_N_ELEMENTS(base32_map); i++) { - const char *x; - - x = memchr(base32_alphabet, ascii_toupper(i), - sizeof base32_alphabet); - base32_map[i] = x ? (x - base32_alphabet) : (unsigned char) -1; - } - } - - ZERO(&s); - si = 0; - i = 0; - - while (i < len) { - unsigned char c; - - c = p[i++]; - if ('=' == c) { - pad++; - c = 0; - } else { - c = base32_map[c]; - if ((unsigned char) -1 == c) { - return -1; - } - } - - s[si++] = c; - - if (G_N_ELEMENTS(s) == si || pad > 0 || i == len) { - char b[5]; - size_t bi; - - memset(&s[si], 0, G_N_ELEMENTS(s) - si); - si = 0; - - b[0] = - ((s[0] << 3) & 0xf8) | - ((s[1] >> 2) & 0x07); - b[1] = - ((s[1] & 0x03) << 6) | - ((s[2] & 0x1f) << 1) | - ((s[3] >> 4) & 1); - b[2] = - ((s[3] & 0x0f) << 4) | - ((s[4] >> 1) & 0x0f); - b[3] = - ((s[4] & 1) << 7) | - ((s[5] & 0x1f) << 2) | - ((s[6] >> 3) & 0x03); - b[4] = - ((s[6] & 0x07) << 5) | - (s[7] & 0x1f); - - for (bi = 0; bi < G_N_ELEMENTS(b) && q != end; bi++) { - *q++ = b[bi]; - } - } - - if (end == q) { - break; - } - } - - return ptr_diff(q, dst); -} diff --git a/libmy/b32_decode.h b/libmy/b32_decode.h deleted file mode 100644 index bc8d4ad9f..000000000 --- a/libmy/b32_decode.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2015 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef B32_DECODE_H -#define B32_DECODE_H - -size_t base32_decode(void *dst, size_t size, const char *data, size_t len); - -#endif /* B32_DECODE_H */ diff --git a/libmy/b32_encode.c b/libmy/b32_encode.c deleted file mode 100644 index 6487fc440..000000000 --- a/libmy/b32_encode.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * See RFC 4648 for details about Base 32 hex encoding: - * http://tools.ietf.org/html/rfc4648 - */ - -#include -#include - -#include "b32_encode.h" - -static const char base32_alphabet[32] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V' -}; - -/** - * Encode in base32 `len' bytes of `data' into the buffer `dst'. - * - * @param dst destination buffer - * @param size length of destination - * @param data start of data to encode - * @param len amount of bytes to encode - * - * @return the amount of bytes generated into the destination. - */ -size_t -base32_encode(char *dst, size_t size, const void *data, size_t len) -{ - size_t i = 0; - const uint8_t *p = data; - const char *end = &dst[size]; - char *q = dst; - - do { - size_t j, k; - uint8_t x[5]; - char s[8]; - - switch (len - i) { - case 4: - k = 7; - break; - case 3: - k = 5; - break; - case 2: - k = 3; - break; - case 1: - k = 2; - break; - default: - k = 8; - } - - for (j = 0; j < 5; j++) - x[j] = i < len ? p[i++] : 0; - - s[0] = (x[0] >> 3); - s[1] = ((x[0] & 0x07) << 2) | (x[1] >> 6); - s[2] = (x[1] >> 1) & 0x1f; - s[3] = ((x[1] & 0x01) << 4) | (x[2] >> 4); - s[4] = ((x[2] & 0x0f) << 1) | (x[3] >> 7); - s[5] = (x[3] >> 2) & 0x1f; - s[6] = ((x[3] & 0x03) << 3) | (x[4] >> 5); - s[7] = x[4] & 0x1f; - - for (j = 0; j < k && q != end; j++) { - *q++ = base32_alphabet[(uint8_t) s[j]]; - } - - if (end == q) { - break; - } - - } while (i < len); - - return q - dst; -} diff --git a/libmy/b32_encode.h b/libmy/b32_encode.h deleted file mode 100644 index 2500f3900..000000000 --- a/libmy/b32_encode.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* - * Copyright (c) 2006 Christian Biere - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the authors nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef B32_ENCODE_H -#define B32_ENCODE_H - -size_t base32_encode(char *dst, size_t dst_len, const void *src, size_t src_len); - -#endif /* B32_ENCODE_H */ diff --git a/libmy/heap.c b/libmy/heap.c deleted file mode 100644 index 1d8f7ee67..000000000 --- a/libmy/heap.c +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2012 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "my_alloc.h" -#include "heap.h" -#include "vector.h" - -VECTOR_GENERATE(ptrvec, void *); - -struct heap { - ptrvec *vec; - heap_compare_func cmp; -}; - -static inline int -cmp_wrapper(heap_compare_func cmp, const void *a, const void *b) -{ - return ((cmp(a, b) < 0) ? 1 : 0); -} - -struct heap * -heap_init(heap_compare_func cmp) -{ - struct heap *h = my_calloc(1, sizeof(*h)); - h->cmp = cmp; - h->vec = ptrvec_init(1); - return (h); -} - -void -heap_destroy(struct heap **h) -{ - if (*h != NULL) { - ptrvec_destroy(&(*h)->vec); - free(*h); - *h = NULL; - } -} - -static int -siftdown(struct heap *h, size_t startpos, size_t pos) -{ - assert(pos < ptrvec_size(h->vec)); - void *newitem = ptrvec_value(h->vec, pos); - while (pos > startpos) { - size_t parentpos = (pos - 1) >> 1; - void *parent = ptrvec_value(h->vec, parentpos); - int cmp = cmp_wrapper(h->cmp, newitem, parent); - if (cmp == -1) - return (-1); - if (cmp == 0) - break; - ptrvec_data(h->vec)[pos] = parent; - pos = parentpos; - } - ptrvec_data(h->vec)[pos] = newitem; - return (0); -} - -static int -siftup(struct heap *h, size_t pos) -{ - assert(pos < ptrvec_size(h->vec)); - void *newitem = ptrvec_value(h->vec, pos); - size_t endpos = ptrvec_size(h->vec); - size_t startpos = pos; - size_t childpos = 2 * pos + 1; - while (childpos < endpos) { - size_t rightpos = childpos + 1; - if (rightpos < endpos) { - int cmp = cmp_wrapper(h->cmp, - ptrvec_value(h->vec, childpos), - ptrvec_value(h->vec, rightpos)); - if (cmp == -1) - return (-1); - if (cmp == 0) - childpos = rightpos; - } - ptrvec_data(h->vec)[pos] = ptrvec_value(h->vec, childpos); - pos = childpos; - childpos = 2 * pos + 1; - } - ptrvec_data(h->vec)[pos] = newitem; - return (siftdown(h, startpos, pos)); -} - -void -heap_push(struct heap *h, void *item) -{ - ptrvec_add(h->vec, item); - siftdown(h, 0, ptrvec_size(h->vec) - 1); -} - -void * -heap_pop(struct heap *h) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - void *returnitem; - void *lastelt = ptrvec_value(h->vec, ptrvec_size(h->vec) - 1); - ptrvec_clip(h->vec, ptrvec_size(h->vec) - 1); - if (ptrvec_size(h->vec) > 0) { - returnitem = ptrvec_value(h->vec, 0); - ptrvec_data(h->vec)[0] = lastelt; - siftup(h, 0); - } else { - returnitem = lastelt; - } - return (returnitem); -} - -void * -heap_replace(struct heap *h, void *item) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - void *returnitem = ptrvec_value(h->vec, 0); - ptrvec_data(h->vec)[0] = item; - siftup(h, 0); - return (returnitem); -} - -void * -heap_peek(struct heap *h) -{ - if (ptrvec_size(h->vec) < 1) - return (NULL); - return ptrvec_data(h->vec)[0]; -} - -void * -heap_get(struct heap *h, size_t i) -{ - if (i > ptrvec_size(h->vec) - 1) - return (NULL); - return (ptrvec_value(h->vec, i)); -} - -size_t -heap_size(struct heap *h) -{ - return (ptrvec_size(h->vec)); -} diff --git a/libmy/heap.h b/libmy/heap.h deleted file mode 100644 index dcd0b0c1c..000000000 --- a/libmy/heap.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef MY_HEAP_H -#define MY_HEAP_H - -struct heap; - -typedef int (*heap_compare_func)(const void *a, const void *b); - -struct heap *heap_init(heap_compare_func); -void heap_destroy(struct heap **); -void heap_push(struct heap *, void *); -void *heap_pop(struct heap *); -void *heap_replace(struct heap *, void *); -void *heap_peek(struct heap *); -void *heap_get(struct heap *, size_t); -size_t heap_size(struct heap *); - -#endif /* MY_HEAP_H */ diff --git a/libmy/my_cpu.h b/libmy/my_cpu.h new file mode 100644 index 000000000..8f789ddf9 --- /dev/null +++ b/libmy/my_cpu.h @@ -0,0 +1,31 @@ +#ifndef MY_CPU_H +#define MY_CPU_H + +#include + +#ifdef __linux__ +# include +#endif /* __linux__ */ + +/* Cores this process may actually run on. Never less than one. */ +static inline long +my_ncpu(void) +{ + long ncpu = -1; + +#ifdef __linux__ + cpu_set_t set; + + if (sched_getaffinity(0, sizeof(set), &set) == 0) + ncpu = CPU_COUNT(&set); +#endif /* __linux__ */ + + if (ncpu < 1) + ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if (ncpu < 1) + ncpu = 1; + + return (ncpu); +} + +#endif /* MY_CPU_H */ diff --git a/libmy/spooldir.c b/libmy/spooldir.c deleted file mode 100644 index 6c7306dec..000000000 --- a/libmy/spooldir.c +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2012 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "my_alloc.h" -#include "spooldir.h" -#include "ubuf.h" - -#define UBUFSZ 128 - -struct spooldir { - pthread_mutex_t lock; - DIR *dir; - int dir_fd; - ubuf *fname; - ubuf *dname_active; - ubuf *dname_incoming; -}; - -static bool -path_exists(const char *path) -{ - struct stat sb; - int ret; - - ret = stat(path, &sb); - if (ret < 0) - return (false); - return (true); -} - -static bool -path_isdir(const char *path) -{ - struct stat sb; - int ret; - - ret = stat(path, &sb); - if (ret < 0) - return (false); - if (S_ISDIR(sb.st_mode)) - return (true); - return (false); -} - -static bool -path_mkdir(const char *path, mode_t mode) -{ - int ret; - - if (path_isdir(path)) { - return (true); - } else { - ret = mkdir(path, mode); - if (ret < 0) { - perror("mkdir"); - return (false); - } - } - return (true); -} - -struct spooldir * -spooldir_init(const char *path) -{ - struct spooldir *s = my_calloc(1, sizeof(*s)); - bool res; - char *dname; - - pthread_mutex_init(&s->lock, NULL); - - dname = realpath(path, NULL); - assert(dname != NULL); - - assert(path_isdir(dname)); - - s->dname_active = ubuf_init(UBUFSZ); - ubuf_add_cstr(s->dname_active, dname); - ubuf_add_cstr(s->dname_active, "/active"); - res = path_mkdir(ubuf_cstr(s->dname_active), 0755); - assert(res); - - s->dname_incoming = ubuf_init(UBUFSZ); - ubuf_add_cstr(s->dname_incoming, dname); - ubuf_add_cstr(s->dname_incoming, "/incoming"); - res = path_mkdir(ubuf_cstr(s->dname_incoming), 0755); - assert(res); - - free(dname); - - s->dir = opendir(ubuf_cstr(s->dname_incoming)); - assert(s->dir != NULL); - - s->dir_fd = dirfd(s->dir); - assert(s->dir_fd != -1); - - s->fname = ubuf_init(UBUFSZ); - - return (s); -} - -void -spooldir_destroy(struct spooldir **s) -{ - if (*s != NULL) { - pthread_mutex_destroy(&(*s)->lock); - closedir((*s)->dir); - ubuf_destroy(&(*s)->fname); - ubuf_destroy(&(*s)->dname_active); - ubuf_destroy(&(*s)->dname_incoming); - free(*s); - *s = NULL; - } -} - -char * -spooldir_next(struct spooldir *s) -{ - struct stat sb; - struct dirent *de; - char *ret = NULL; - size_t retsz; - char *fname = NULL; - ubuf *src_fname; - - pthread_mutex_lock(&s->lock); - - while (fname == NULL) { - while ((de = readdir(s->dir)) != NULL) { - if (de->d_name[0] == '.') - continue; - if (fstatat(s->dir_fd, de->d_name, &sb, 0) == -1) { - fprintf(stderr, "%s: fstatat() failed: %s\n", - __func__, strerror(errno)); - continue; - } - if (!S_ISREG(sb.st_mode)) - continue; - fname = de->d_name; - break; - } - - if (fname == NULL) { - rewinddir(s->dir); - usleep(500*1000); - pthread_mutex_unlock(&s->lock); - usleep(500*1000); - return (NULL); - } - } - - assert(fname != NULL); - - src_fname = ubuf_init(UBUFSZ); - ubuf_extend(src_fname, s->dname_incoming); - ubuf_add_fmt(src_fname, "/%s", fname); - ubuf_cterm(src_fname); - - ubuf_clip(s->fname, 0); - ubuf_extend(s->fname, s->dname_active); - ubuf_add_fmt(s->fname, "/%s", fname); - ubuf_cterm(s->fname); - - if (path_exists(ubuf_cstr(s->fname))) { - fprintf(stderr, "%s: WARNING: unlinking destination path %s\n", - __func__, ubuf_cstr(s->fname)); - unlink(ubuf_cstr(s->fname)); - } - - int rename_ret = rename(ubuf_cstr(src_fname), ubuf_cstr(s->fname)); - if (rename_ret != 0) { - fprintf(stderr, "rename(%s, %s): %s\n", - ubuf_cstr(src_fname), ubuf_cstr(s->fname), strerror(errno)); - goto out; - } - - ubuf_detach(s->fname, (uint8_t **) &ret, &retsz); -out: - ubuf_destroy(&src_fname); - pthread_mutex_unlock(&s->lock); - return (ret); -} diff --git a/libmy/spooldir.h b/libmy/spooldir.h deleted file mode 100644 index 88d146df5..000000000 --- a/libmy/spooldir.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef MY_SPOOLDIR_H -#define MY_SPOOLDIR_H - -struct spooldir; - -struct spooldir *spooldir_init(const char *path); -void spooldir_destroy(struct spooldir **); -char *spooldir_next(struct spooldir *); - -#endif /* MY_SPOOLDIR_H */ diff --git a/libmy/varint.c b/libmy/varint.c deleted file mode 100644 index 696927475..000000000 --- a/libmy/varint.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2012, 2013 by Farsight Security, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -/* - * Copyright (c) 2008, Dave Benson. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "varint.h" - -unsigned -varint_length(uint64_t v) -{ - unsigned len = 1; - while (v >= 128) { - v >>= 7; - len++; - } - return (len); -} - -unsigned -varint_length_packed(const uint8_t *data, size_t len_data) -{ - unsigned i = 0; - size_t len = len_data; - while (len--) { - if ((data[i] & 0x80) == 0) - break; - i++; - } - if (i == len_data) - return (0); - return (i + 1); -} - -size_t -varint_encode32(uint8_t *src_ptr, uint32_t v) -{ - static const unsigned B = 128; - uint8_t *ptr = src_ptr; - if (v < (1 << 7)) { - *(ptr++) = v; - } else if (v < (1 << 14)) { - *(ptr++) = v | B; - *(ptr++) = v >> 7; - } else if (v < (1 << 21)) { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = v >> 14; - } else if (v < (1 << 28)) { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = (v >> 14) | B; - *(ptr++) = v >> 21; - } else { - *(ptr++) = v | B; - *(ptr++) = (v >> 7) | B; - *(ptr++) = (v >> 14) | B; - *(ptr++) = (v >> 21) | B; - *(ptr++) = v >> 28; - } - return ((size_t) (ptr - src_ptr)); -} - -size_t -varint_encode64(uint8_t *src_ptr, uint64_t v) -{ - static const unsigned B = 128; - uint8_t *ptr = src_ptr; - while (v >= B) { - *(ptr++) = (v & (B - 1)) | B; - v >>= 7; - } - *(ptr++) = (uint8_t) v; - return ((size_t) (ptr - src_ptr)); -} - -size_t -varint_decode32(const uint8_t *data, uint32_t *value) -{ - unsigned len = varint_length_packed(data, 5); - uint32_t val = data[0] & 0x7f; - if (len > 1) { - val |= ((data[1] & 0x7f) << 7); - if (len > 2) { - val |= ((data[2] & 0x7f) << 14); - if (len > 3) { - val |= ((data[3] & 0x7f) << 21); - if (len > 4) - val |= (data[4] << 28); - } - } - } - *value = val; - return ((size_t) len); -} - -size_t -varint_decode64(const uint8_t *data, uint64_t *value) -{ - unsigned shift, i; - unsigned len = varint_length_packed(data, 10); - uint64_t val; - if (len < 5) { - size_t tmp_len; - uint32_t tmp; - tmp_len = varint_decode32(data, &tmp); - *value = tmp; - return (tmp_len); - } - val = ((data[0] & 0x7f)) - | ((data[1] & 0x7f) << 7) - | ((data[2] & 0x7f) << 14) - | ((data[3] & 0x7f) << 21); - shift = 28; - for (i = 4; i < len; i++) { - val |= (((uint64_t)(data[i] & 0x7f)) << shift); - shift += 7; - } - *value = val; - return ((size_t) len); -} diff --git a/libmy/varint.h b/libmy/varint.h deleted file mode 100644 index ad91b0cc5..000000000 --- a/libmy/varint.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MY_VARINT_H -#define MY_VARINT_H - -#include -#include - -unsigned varint_length(uint64_t v); -unsigned varint_length_packed(const uint8_t *buf, size_t len_buf); -size_t varint_encode32(uint8_t *ptr, uint32_t value); -size_t varint_encode64(uint8_t *ptr, uint64_t value); -size_t varint_decode32(const uint8_t *ptr, uint32_t *value); -size_t varint_decode64(const uint8_t *ptr, uint64_t *value); - -#endif /* MY_VARINT_H */ diff --git a/libmy/zonefile.c b/libmy/zonefile.c deleted file mode 100644 index eddaa8b1e..000000000 --- a/libmy/zonefile.c +++ /dev/null @@ -1,188 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "my_alloc.h" -#include "ubuf.h" -#include "zonefile.h" - -struct zonefile { - FILE *fp; - bool eof; - bool is_pipe; - bool valid; - ldns_rdf *domain; - ldns_rdf *origin; - ldns_rdf *prev; - ldns_rr *rr_soa; - uint32_t ttl; - size_t count; -}; - -static ldns_status -read_soa(struct zonefile *z) -{ - ldns_rr *rr; - ldns_status status; - - for (;;) { - status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); - switch (status) { - case LDNS_STATUS_OK: - goto out; - case LDNS_STATUS_SYNTAX_EMPTY: - case LDNS_STATUS_SYNTAX_TTL: - case LDNS_STATUS_SYNTAX_ORIGIN: - status = LDNS_STATUS_OK; - break; - default: - goto out; - } - } -out: - if (status != LDNS_STATUS_OK) { - z->valid = false; - return (LDNS_STATUS_ERR); - } - - if (ldns_rr_get_type(rr) != LDNS_RR_TYPE_SOA) { - ldns_rr_free(rr); - z->valid = false; - return (LDNS_STATUS_ERR); - } - - z->count = 1; - z->domain = ldns_rdf_clone(ldns_rr_owner(rr)); - z->origin = ldns_rdf_clone(ldns_rr_owner(rr)); - z->rr_soa = rr; - return (LDNS_STATUS_OK); -} - -struct zonefile * -zonefile_init_fname(const char *fname) -{ - struct zonefile *z = my_calloc(1, sizeof(struct zonefile)); - - size_t len_fname = strlen(fname); - if (len_fname >= 3 && - fname[len_fname - 3] == '.' && - fname[len_fname - 2] == 'g' && - fname[len_fname - 1] == 'z') - { - ubuf *u = ubuf_new(); - ubuf_add_cstr(u, "zcat "); - ubuf_add_cstr(u, fname); - z->fp = popen(ubuf_cstr(u), "r"); - z->is_pipe = true; - ubuf_destroy(&u); - } else { - z->fp = fopen(fname, "r"); - } - - if (z->fp == NULL) - return (NULL); - - z->valid = true; - if (read_soa(z) != LDNS_STATUS_OK) - zonefile_destroy(&z); - - return (z); -} - -void -zonefile_destroy(struct zonefile **z) -{ - if (*z) { - if ((*z)->fp) { - if ((*z)->is_pipe) - pclose((*z)->fp); - else - fclose((*z)->fp); - } - if ((*z)->origin) - ldns_rdf_deep_free((*z)->origin); - if ((*z)->prev) - ldns_rdf_deep_free((*z)->prev); - if ((*z)->domain) - ldns_rdf_deep_free((*z)->domain); - if ((*z)->rr_soa) - ldns_rr_free((*z)->rr_soa); - free(*z); - *z = NULL; - } -} - -const ldns_rdf * -zonefile_get_domain(struct zonefile *z) -{ - return (z->domain); -} - -size_t -zonefile_get_count(struct zonefile *z) -{ - return (z->count); -} - -uint32_t -zonefile_get_serial(struct zonefile *z) -{ - ldns_rdf *rdf = ldns_rr_rdf(z->rr_soa, 2); - assert(rdf != NULL); - return (ldns_rdf2native_int32(rdf)); -} - -ldns_status -zonefile_read(struct zonefile *z, ldns_rr **out) -{ - ldns_rr *rr; - ldns_status status = LDNS_STATUS_OK; - - if (z->eof) { - *out = NULL; - return (LDNS_STATUS_OK); - } - - if (!z->valid) - return (LDNS_STATUS_ERR); - - if (z->count == 1 && z->rr_soa != NULL) { - *out = z->rr_soa; - z->rr_soa = NULL; - return (LDNS_STATUS_OK); - } - for (;;) { - if (feof(z->fp)) { - *out = NULL; - z->eof = true; - return (LDNS_STATUS_OK); - } - status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); - switch (status) { - case LDNS_STATUS_OK: - if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_SOA) { - ldns_rr_free(rr); - *out = NULL; - return (LDNS_STATUS_OK); - } - z->count++; - goto out; - case LDNS_STATUS_SYNTAX_EMPTY: - case LDNS_STATUS_SYNTAX_TTL: - case LDNS_STATUS_SYNTAX_ORIGIN: - status = LDNS_STATUS_OK; - break; - default: - goto out; - } - } -out: - if (status != LDNS_STATUS_OK) - return (status); - *out = rr; - return (status); -} diff --git a/libmy/zonefile.h b/libmy/zonefile.h deleted file mode 100644 index f6d78562e..000000000 --- a/libmy/zonefile.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef MY_ZONEFILE_H -#define MY_ZONEFILE_H - -#include - -struct zonefile; - -struct zonefile * -zonefile_init_fname(const char *fname); - -void -zonefile_destroy(struct zonefile **); - -const ldns_rdf * -zonefile_get_domain(struct zonefile *); - -size_t -zonefile_get_count(struct zonefile *); - -uint32_t -zonefile_get_serial(struct zonefile *); - -ldns_status -zonefile_read(struct zonefile *, ldns_rr **); - -#endif /* MY_ZONEFILE_H */ diff --git a/nmsg/alias.c b/nmsg/alias.c index eae9ee919..03168853c 100644 --- a/nmsg/alias.c +++ b/nmsg/alias.c @@ -169,6 +169,10 @@ alias_init(struct nmsg_alias *al, const char *fname) { } al->value[key] = strdup(str_value); + if (al->value[key] == NULL) { + res = nmsg_res_failure; + break; + } } fclose(fp); diff --git a/nmsg/base/dnsqr.c b/nmsg/base/dnsqr.c index 103754c7a..d75bcc2d5 100644 --- a/nmsg/base/dnsqr.c +++ b/nmsg/base/dnsqr.c @@ -773,6 +773,7 @@ get_af(const char *addr) { static char * addrs_to_bpf(const char *addrs, const char *bpfdir, int af) { char *ret, *tok_addrs, *addr, *saveptr; + uint8_t *ubuf_str; size_t retsz; int addr_af; ubuf *bpf; @@ -815,8 +816,9 @@ addrs_to_bpf(const char *addrs, const char *bpfdir, int af) { free(tok_addrs); ubuf_cterm(bpf); - ubuf_detach(bpf, (uint8_t **) &ret, &retsz); + ubuf_detach(bpf, &ubuf_str, &retsz); ubuf_destroy(&bpf); + ret = (char *)ubuf_str; return (ret); } diff --git a/nmsg/base/encode.c b/nmsg/base/encode.c index 8742800b0..49974b59f 100644 --- a/nmsg/base/encode.c +++ b/nmsg/base/encode.c @@ -18,9 +18,7 @@ */ /* Import. */ -#ifdef HAVE_JSON_C #include -#endif #include "encode.pb-c.h" @@ -66,7 +64,6 @@ encode_payload_add_value(struct nmsg_strbuf *sb, int type_value, const char *dat /* validate json */ if (is_json) { -#ifdef HAVE_JSON_C struct json_tokener *jtok = json_tokener_new(); struct json_object *jobj; @@ -79,9 +76,6 @@ encode_payload_add_value(struct nmsg_strbuf *sb, int type_value, const char *dat json_tokener_free(jtok); if (jobj == NULL) return false; -#else - return false; -#endif } declare_json_value(sb, "val", true); diff --git a/nmsg/chalias.c b/nmsg/chalias.c index 870820832..d1c72b385 100644 --- a/nmsg/chalias.c +++ b/nmsg/chalias.c @@ -61,13 +61,28 @@ nmsg_chalias_lookup(const char *ch, char ***alias) { while (fgets(line, sizeof(line), fp) != NULL) { tmp = strtok_r(line, " \t", &saveptr); if (tmp != NULL && strcmp(tmp, ch) == 0) { - while ((tmp = strtok_r(NULL, " \t\n", &saveptr)) - != NULL) - { + while ((tmp = strtok_r(NULL, " \t\n", &saveptr)) != NULL) { + char **ptr, *dup; + num_aliases += 1; - *alias = realloc(*alias, - sizeof(**alias) * num_aliases); - (*alias)[num_aliases - 1] = strdup(tmp); + + ptr = realloc(*alias, sizeof(**alias) * num_aliases); + if (ptr == NULL) { + fclose(fp); + return (-1); + } + + *alias = ptr; + + dup = strdup(tmp); + + // terminate the alias buffer whether strdup(3) succeeds or not + (*alias)[num_aliases - 1] = dup; + + if (dup == NULL) { + fclose(fp); + return (-1); + } } } } @@ -75,7 +90,14 @@ nmsg_chalias_lookup(const char *ch, char ***alias) { fclose(fp); /* append NULL sentinel */ - *alias = realloc(*alias, sizeof(**alias) * (num_aliases + 1)); + char **ptr = realloc(*alias, sizeof(**alias) * (num_aliases + 1)); + if (ptr == NULL && *alias != NULL) { + free((*alias)[num_aliases - 1]); + (*alias)[num_aliases - 1] = NULL; + return (-1); + } + + *alias = ptr; (*alias)[num_aliases] = NULL; return (num_aliases); diff --git a/nmsg/chalias.h b/nmsg/chalias.h index 73b9164be..eef7e2c2d 100644 --- a/nmsg/chalias.h +++ b/nmsg/chalias.h @@ -28,7 +28,8 @@ * * \param[out] alias Location to store an array of sockspecs. * - * \return Number of aliases. + * \return Number of aliases or -1 if an error has occurred, in which case + * the caller must invoke nmsg_chalias_free() for the alias. */ int nmsg_chalias_lookup(const char *ch, char ***alias); diff --git a/nmsg/dlmod.c b/nmsg/dlmod.c index c316523d9..6d23a6676 100644 --- a/nmsg/dlmod.c +++ b/nmsg/dlmod.c @@ -35,6 +35,7 @@ _nmsg_dlmod_init(const char *path) { dlmod->handle = dlopen(path, RTLD_LAZY); if (dlmod->handle == NULL) { _nmsg_dprintf(1, "%s: %s\n", __func__, dlerror()); + free(dlmod->path); free(dlmod); return (NULL); } diff --git a/nmsg/input.c b/nmsg/input.c index 1bfc1066d..ac32c5d83 100644 --- a/nmsg/input.c +++ b/nmsg/input.c @@ -38,7 +38,7 @@ nmsg_input_open_sock(int fd) { return (input_open_stream(nmsg_stream_type_sock, fd)); } -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) nmsg_input_t nmsg_input_open_kafka_json(const char *address) { @@ -66,12 +66,47 @@ nmsg_input_open_kafka_json(const char *address) return (input); } -#else /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#else /* (defined HAVE_LIBRDKAFKA) */ nmsg_input_t nmsg_input_open_kafka_json(const char *address __attribute__((unused))) { return (NULL); } -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ + +#ifdef HAVE_LIBRDKAFKA +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address) +{ + struct nmsg_input *input; + + input = calloc(1, sizeof(*input)); + if (input == NULL) + return (NULL); + + input->kafka = calloc(1, sizeof(*(input->kafka))); + if (input->kafka == NULL) { + free(input); + return (NULL); + } + + input->type = nmsg_input_type_kafka_payload; + input->read_fp = _input_kafka_payload_read; + + input->kafka->ctx = kafka_create_consumer(address, NMSG_RBUF_TIMEOUT); + if (input->kafka->ctx == NULL) { + free(input->kafka); + free(input); + return (NULL); + } + + return (input); +} +#else /* HAVE_LIBRDKAFKA */ +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address __attribute__((unused))) { + return (NULL); +} +#endif /* HAVE_LIBRDKAFKA */ #ifdef HAVE_LIBRDKAFKA nmsg_input_t @@ -178,7 +213,6 @@ nmsg_input_open_pres(int fd, nmsg_msgmod_t msgmod) { return (input); } -#ifdef HAVE_JSON_C nmsg_input_t nmsg_input_open_json(int fd) { struct nmsg_input *input; @@ -214,12 +248,6 @@ nmsg_input_open_json(int fd) { return (input); } -#else /* HAVE_JSON_C */ -nmsg_input_t -nmsg_input_open_json(__attribute__((unused)) int fd) { - return (NULL); -} -#endif /* HAVE_JSON_C */ nmsg_input_t nmsg_input_open_pcap(nmsg_pcap_t pcap, nmsg_msgmod_t msgmod) { @@ -295,6 +323,7 @@ nmsg_input_close(nmsg_input_t *input) { free((*input)->json); break; case nmsg_input_type_kafka_json: + case nmsg_input_type_kafka_payload: #ifdef HAVE_LIBRDKAFKA kafka_ctx_destroy(&(*input)->kafka->ctx); free((*input)->kafka); diff --git a/nmsg/input.h b/nmsg/input.h index afefe2613..35afda285 100644 --- a/nmsg/input.h +++ b/nmsg/input.h @@ -59,6 +59,7 @@ typedef enum { nmsg_input_type_callback, nmsg_input_type_json, /*%< JSON form */ nmsg_input_type_kafka_json, /*%< NMSG payloads from Kafka in JSON form */ + nmsg_input_type_kafka_payload, /*%< NMSG single binary payloads from Kafka */ } nmsg_input_type; /** @@ -133,9 +134,9 @@ nmsg_input_open_zmq_endpoint(void *zmq_ctx, const char *ep); * Only if a partition number has been specified can offset be a numeric value. * Note that only new consumer group IDs will honor these directives. * - * The value of proto must be either "nmsg" (binary container input) or "json" - * (JSON-serialized payloads) and either or both a partition number and offset - * value may be optionally supplied. + * The value of proto must be "nmsg" (binary container input), "nmsgp" (single + * payload input), or "json" (JSON-serialized payloads) and either or both a + * partition number and offset value may be optionally supplied. * * \see nmsg_output_open_kafka_endpoint() * @@ -214,6 +215,21 @@ nmsg_input_open_json(int fd); nmsg_input_t nmsg_input_open_kafka_json(const char *address); +/** + * Initialize a new NMSG single payload input from a Kafka broker. + * + * Each Kafka message is expected to contain a single serialized + * NmsgPayload protobuf (not a full NMSG container). + * + * See nmsg_input_open_kafka_endpoint for the details of the address string. + * + * \param[in] address Kafka endpoint address string. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_input_t +nmsg_input_open_kafka_payload(const char *address); + /** * Initialize a new NMSG pcap input from a pcap descriptor. diff --git a/nmsg/input_json.c b/nmsg/input_json.c index 2c377c063..1e5d227e9 100644 --- a/nmsg/input_json.c +++ b/nmsg/input_json.c @@ -21,23 +21,26 @@ /* Internal functions. */ -#if (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) +#if (defined HAVE_LIBRDKAFKA) nmsg_res _input_kafka_json_read(nmsg_input_t input, nmsg_message_t *msg) { + uint8_t *ubuf_str; nmsg_res res; char *buf; size_t buf_len; - res = kafka_read_start(input->kafka->ctx, (uint8_t **) &buf, &buf_len); + res = kafka_read_start(input->kafka->ctx, &ubuf_str, &buf_len); if (res != nmsg_res_success) { kafka_read_finish(input->kafka->ctx); return res; } + buf = (char *)ubuf_str; + if (buf_len == 0) return nmsg_res_failure; - res = nmsg_message_from_json((const char *) buf, msg); + res = nmsg_message_from_json((const char *)buf, msg); if (res == nmsg_res_parse_error) { _nmsg_dprintf(2, "Kafka JSON parse error: \"%s\"\n", buf); @@ -47,9 +50,8 @@ _input_kafka_json_read(nmsg_input_t input, nmsg_message_t *msg) { kafka_read_finish(input->kafka->ctx); return res; } -#endif /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ -#ifdef HAVE_JSON_C nmsg_res _input_json_read(nmsg_input_t input, nmsg_message_t *msg) { char line[1024]; @@ -94,10 +96,3 @@ _input_json_read(nmsg_input_t input, nmsg_message_t *msg) { _nmsg_strbuf_destroy(&sbs); return (nmsg_res_eof); } -#else /* HAVE_JSON_C */ -nmsg_res -_input_json_read(__attribute__((unused)) nmsg_input_t input, - __attribute__((unused)) nmsg_message_t *msg) { - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ diff --git a/nmsg/input_nmsg.c b/nmsg/input_nmsg.c index e67e9c916..839098eef 100644 --- a/nmsg/input_nmsg.c +++ b/nmsg/input_nmsg.c @@ -156,7 +156,7 @@ _input_nmsg_filter(nmsg_input_t input, unsigned idx, Nmsg__NmsgPayload *np) { uint32_t calc_crc = my_crc32c(np->payload.data, np->payload.len); if (ntohl(wire_crc) != calc_crc) { _nmsg_dprintf(1, "libnmsg: WARNING: crc mismatch (%x != %x) [%s]\n", - calc_crc, wire_crc, __func__); + calc_crc, ntohl(wire_crc), __func__); return (false); } } @@ -390,7 +390,7 @@ _input_process_buffer_into_container(nmsg_input_t input, Nmsg__Nmsg **nmsg, uint /* expire old outstanding fragments */ _input_frag_gc(input->stream); - return nmsg_res_success; + return (res); } #endif /* defined(HAVE_LIBRDKAFKA) || defined(HAVE_LIBZMQ) */ @@ -416,6 +416,53 @@ _input_nmsg_read_container_kafka(nmsg_input_t input, Nmsg__Nmsg **nmsg) { } #endif /* HAVE_LIBRDKAFKA */ +#ifdef HAVE_LIBRDKAFKA +nmsg_res +_input_kafka_payload_read(nmsg_input_t input, nmsg_message_t *msg) { + nmsg_res res; + uint8_t *buf; + size_t buf_len; + Nmsg__NmsgPayload *np; + + res = kafka_read_start(input->kafka->ctx, &buf, &buf_len); + if (res != nmsg_res_success) { + kafka_read_finish(input->kafka->ctx); + return res; + } + + if (buf_len == 0) { + kafka_read_finish(input->kafka->ctx); + return nmsg_res_failure; + } + + np = nmsg__nmsg_payload__unpack(NULL, buf_len, buf); + + kafka_read_finish(input->kafka->ctx); + + if (np == NULL) { + _nmsg_dprintf(1, "%s: failed to unpack payload\n", __func__); + return nmsg_res_parse_error; + } + + /* filter (vid, msgtype) */ + if (input->do_filter && + (input->filter_vid != np->vid || + input->filter_msgtype != np->msgtype)) + { + _nmsg_payload_free(&np); + return nmsg_res_again; + } + + *msg = _nmsg_message_from_payload(np); + if (*msg == NULL) { + _nmsg_payload_free(&np); + return nmsg_res_memfail; + } + + return nmsg_res_success; +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBZMQ nmsg_res _input_nmsg_read_container_zmq(nmsg_input_t input, Nmsg__Nmsg **nmsg) { diff --git a/nmsg/msgmod/message.c b/nmsg/msgmod/message.c index 8110d48f7..34c94aa9d 100644 --- a/nmsg/msgmod/message.c +++ b/nmsg/msgmod/message.c @@ -216,7 +216,6 @@ nmsg_message_from_raw_payload(unsigned vid, unsigned msgtype, return (msg); } -#ifdef HAVE_JSON_C nmsg_res nmsg_message_from_json(const char *json, nmsg_message_t *msg) { nmsg_res res = nmsg_res_parse_error; @@ -361,13 +360,6 @@ nmsg_message_from_json(const char *json, nmsg_message_t *msg) { json_object_put(node); return (res); } -#else /* HAVE_JSON_C */ -nmsg_res -nmsg_message_from_json(__attribute__((unused)) const char *json, - __attribute__((unused)) nmsg_message_t *msg) { - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ nmsg_res _nmsg_message_init_message(struct nmsg_message *msg) { diff --git a/nmsg/msgmod/transparent_json.c b/nmsg/msgmod/transparent_json.c index 08e571166..df3c3146d 100644 --- a/nmsg/msgmod/transparent_json.c +++ b/nmsg/msgmod/transparent_json.c @@ -19,7 +19,6 @@ #include "transparent.h" -#ifdef HAVE_JSON_C nmsg_res _nmsg_msgmod_json_to_message(void *val, struct nmsg_message *msg) { struct json_object *node = (struct json_object *)val; @@ -321,21 +320,3 @@ _nmsg_msgmod_json_to_payload_load(struct nmsg_message *msg, return nmsg_res_failure; } - -#else /* HAVE_JSON_C */ -nmsg_res -_nmsg_msgmod_json_to_message(__attribute__((unused)) void *val, - __attribute__((unused)) struct nmsg_message *msg) { - return (nmsg_res_notimpl); -} - -nmsg_res -_nmsg_msgmod_json_to_payload_load(__attribute__((unused)) struct nmsg_message *msg, - __attribute__((unused)) struct nmsg_msgmod_field *field, - __attribute__((unused)) unsigned field_idx, - __attribute__((unused)) unsigned val_idx, - __attribute__((unused)) void *val) -{ - return (nmsg_res_notimpl); -} -#endif /* HAVE_JSON_C */ diff --git a/nmsg/output.c b/nmsg/output.c index 5321f8193..19c499a39 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -64,8 +64,15 @@ nmsg_output_open_kafka_json(const char *addr, const char *key_field) return NULL; } - if (key_field != NULL) + if (key_field != NULL) { output->kafka->key_field = strdup(key_field); + if (output->kafka->key_field == NULL) { + kafka_ctx_destroy(&output->kafka->ctx); + free(output->kafka); + free(output); + return (NULL); + } + } return output; }; @@ -78,6 +85,47 @@ nmsg_output_open_kafka_json(const char *addr __attribute__((unused)), } #endif /* HAVE_LIBRDKAFKA */ +#ifdef HAVE_LIBRDKAFKA +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr, const char *key_field) +{ + struct nmsg_output *output; + + output = calloc(1, sizeof(*output)); + if (output == NULL) + return (NULL); + + output->kafka = calloc(1, sizeof(*(output->kafka))); + if (output->kafka == NULL) { + free(output); + return (NULL); + } + + output->type = nmsg_output_type_kafka_payload; + output->write_fp = _output_kafka_payload_write; + output->flush_fp = _output_kafka_payload_flush; + + output->kafka->ctx = kafka_create_producer(addr, NMSG_RBUF_TIMEOUT); + if (!output->kafka->ctx) { + free(output->kafka); + free(output); + return (NULL); + } + + if (key_field != NULL) + output->kafka->key_field = strdup(key_field); + + return output; +} +#else /* HAVE_LIBRDKAFKA */ +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr __attribute__((unused)), + const char *key_field __attribute__((unused))) +{ + return (NULL); +} +#endif /* HAVE_LIBRDKAFKA */ + #ifdef HAVE_LIBRDKAFKA nmsg_output_t _output_open_kafka(void *s, size_t bufsz) { @@ -137,6 +185,12 @@ nmsg_output_open_pres(int fd) { return (NULL); } output->pres->endline = strdup("\n"); + if (output->pres->endline == NULL) { + fclose(output->pres->fp); + free(output->pres); + free(output); + return (NULL); + } pthread_mutex_init(&output->pres->lock, NULL); return (output); @@ -226,12 +280,21 @@ nmsg_output_write(nmsg_output_t output, nmsg_message_t msg) { nmsg_res nmsg_output_close(nmsg_output_t *output) { - nmsg_res res; + nmsg_res res, async_res; res = nmsg_res_success; switch ((*output)->type) { case nmsg_output_type_stream: res = _output_nmsg_flush(*output); + + /* + * Before random, fd and the locks below, all of which the + * compressor thread uses. + */ + async_res = _output_async_destroy(*output); + if (res == nmsg_res_success) + res = async_res; + if ((*output)->stream->random != NULL) nmsg_random_destroy(&((*output)->stream->random)); #ifdef HAVE_LIBRDKAFKA @@ -253,6 +316,7 @@ nmsg_output_close(nmsg_output_t *output) { close((*output)->stream->fd); } nmsg_container_destroy(&(*output)->stream->c); + pthread_cond_destroy(&(*output)->stream->c_drained); pthread_mutex_destroy(&(*output)->stream->c_lock); pthread_mutex_destroy(&(*output)->stream->w_lock); free((*output)->stream); @@ -269,6 +333,7 @@ nmsg_output_close(nmsg_output_t *output) { free((*output)->json); break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: #ifdef HAVE_LIBRDKAFKA kafka_ctx_destroy(&(*output)->kafka->ctx); if ((*output)->kafka->key_field != NULL) @@ -276,6 +341,7 @@ nmsg_output_close(nmsg_output_t *output) { free((*output)->kafka); #else /* HAVE_LIBRDKAFKA */ assert((*output)->type != nmsg_output_type_kafka_json); + assert((*output)->type != nmsg_output_type_kafka_payload); #endif /* HAVE_LIBRDKAFKA */ break; case nmsg_output_type_callback: @@ -353,12 +419,78 @@ nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout) { output->stream->do_zlib = zlibout; } +nmsg_res +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers) +{ + /* + * Type test first: 'stream' is a union member, so reading stream->type + * on a pres or json output would reinterpret another struct's bytes. + */ + if (output->type != nmsg_output_type_stream) + return (nmsg_res_success); + if (output->stream->type != nmsg_stream_type_file) + return (nmsg_res_success); + + /* + * Unbuffered flushes a container per message, so a pool would spend a + * ticket and a wakeup per message to compress a single payload. + */ + if (workers > 0 && !output->stream->buffered) + return (nmsg_res_failure); + + if (workers > 0) + return (_output_async_init(output, workers)); + + /* Turning a pool off can strand an error a worker recorded. */ + return (_output_async_destroy(output)); +} + +void +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, + unsigned idle_secs) +{ + struct nmsg_stream_output *ostr; + struct nmsg_ostr_async *pool; + + /* Type test first, for the reason nmsg_output_set_zlib_workers() gives. */ + if (output->type != nmsg_output_type_stream) + return; + if (output->stream->type != nmsg_stream_type_file) + return; + + ostr = output->stream; + + /* + * Kept on the stream, not just in the pool: a ceiling change replaces + * the pool, and this way the two setters may be called in any order. + * Nothing is logged when there is no pool -- nmsgtool sets a policy on + * every output, so it would fire on a plain '--unbuffered -w'. + */ + pthread_mutex_lock(&ostr->c_lock); + ostr->so_zmin = min_workers; + ostr->so_zcull = idle_secs; + + /* + * Taken under c_lock so a teardown cannot free the pool underneath. + * Lock order is c_lock then pool->lock; nothing takes them the other way. + */ + pool = _output_async_ref(ostr); + pthread_mutex_unlock(&ostr->c_lock); + + if (pool != NULL) { + _output_async_set_cull(pool, min_workers, idle_secs); + _output_async_unref(ostr); + } +} + void nmsg_output_set_endline(nmsg_output_t output, const char *endline) { if (output->type == nmsg_output_type_pres) { - if (output->pres->endline != NULL) + char *ptr = strdup(endline); + if (ptr != NULL) { free(output->pres->endline); - output->pres->endline = strdup(endline); + output->pres->endline = ptr; + } } } @@ -375,6 +507,7 @@ nmsg_output_set_source(nmsg_output_t output, unsigned source) { output->json->source = source; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->source = source; default: break; @@ -394,6 +527,7 @@ nmsg_output_set_operator(nmsg_output_t output, unsigned operator) { output->json->operator = operator; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->operator = operator; default: break; @@ -413,6 +547,7 @@ nmsg_output_set_group(nmsg_output_t output, unsigned group) { output->json->group = group; break; case nmsg_output_type_kafka_json: + case nmsg_output_type_kafka_payload: output->kafka->group = group; default: break; @@ -423,11 +558,11 @@ void _output_stop(nmsg_output_t output) { output->stop = true; #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C if (output->type == nmsg_output_type_kafka_json) kafka_stop(output->kafka->ctx); -#endif /* HAVE_JSON_C */ - if (output->type == nmsg_output_type_stream && + if (output->type == nmsg_output_type_kafka_payload) + kafka_stop(output->kafka->ctx); + else if (output->type == nmsg_output_type_stream && output->stream != NULL && output->stream->type == nmsg_stream_type_kafka) kafka_stop(output->stream->kafka); @@ -474,6 +609,8 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { } output->stream->type = type; output->stream->buffered = true; + output->stream->so_zmin = NMSG_ZCULL_MIN_WORKERS_DEFAULT; + output->stream->so_zcull = NMSG_ZCULL_SECS_DEFAULT; /* seed the rng, needed for fragment and sequence IDs */ output->stream->random = nmsg_random_init(); @@ -485,8 +622,21 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { pthread_mutex_init(&output->stream->c_lock, NULL); pthread_mutex_init(&output->stream->w_lock, NULL); + if (pthread_cond_init(&output->stream->c_drained, NULL) != 0) { + nmsg_random_destroy(&output->stream->random); + pthread_mutex_destroy(&output->stream->c_lock); + pthread_mutex_destroy(&output->stream->w_lock); + free(output->stream); + free(output); + return (NULL); + } - /* enable container sequencing */ + /* + * Enable container sequencing. Sock and zmq only, which is what lets + * the async compressor number containers in compression order; + * widening this test needs _output_nmsg_container_compress() looked + * at. + */ if (output->stream->type == nmsg_stream_type_sock || output->stream->type == nmsg_stream_type_zmq) { @@ -509,6 +659,7 @@ output_open_stream_base(nmsg_stream_type type, size_t bufsz) { output->stream->c = nmsg_container_init(bufsz); if (output->stream->c == NULL) { nmsg_random_destroy(&output->stream->random); + pthread_cond_destroy(&output->stream->c_drained); pthread_mutex_destroy(&output->stream->c_lock); pthread_mutex_destroy(&output->stream->w_lock); free(output->stream); diff --git a/nmsg/output.h b/nmsg/output.h index 83a8048f5..c7ebccf81 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -42,6 +42,7 @@ typedef enum { nmsg_output_type_callback, nmsg_output_type_json, nmsg_output_type_kafka_json, + nmsg_output_type_kafka_payload, } nmsg_output_type; /** @@ -193,6 +194,21 @@ nmsg_output_open_json(int fd); nmsg_output_t nmsg_output_open_kafka_json(const char *addr, const char *key_field); +/** + * Initialize a new NMSG binary-payload output to a Kafka broker. + * + * Each Kafka message carries a single serialized binary NMSG Payload protobuf. + * See nmsg_input_open_kafka_endpoint for the details of the address string. + * + * \param[in] addr Kafka endpoint address string (without proto: prefix). + * \param[in] key_field An optional NMSG field name whose content will be + * used as a Kafka producer key. Otherwise, its value should be NULL. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_output_t +nmsg_output_open_kafka_payload(const char *addr, const char *key_field); + /** * Initialize a new nmsg output closure. This allows a user-provided callback to * function as an nmsg output, for instance to participate in an nmsg_io loop. @@ -366,4 +382,53 @@ nmsg_output_set_group(nmsg_output_t output, unsigned group); void nmsg_output_set_zlibout(nmsg_output_t output, bool zlibout); +/** + * Compress containers on worker threads rather than on the thread that filled + * them: a reader that compresses is not reading, and on a busy channel that + * pause is long enough for the socket to overflow. + * + * workers is a ceiling, not an allocation: threads start on demand and are + * given back once idle, see nmsg_output_set_zlib_cull(). Write order is + * unchanged, and a producer that finds every worker busy compresses inline, so + * this is never slower than leaving it off. + * + * A write error surfaces on a later nmsg_output_write(), nmsg_output_flush() or + * nmsg_output_close(). File outputs only, and only when buffered. + * + * Not thread-safe against a concurrent write on the same output: turning a pool + * on or off while another thread is writing can reorder containers. + * + * \param[in] output nmsg_output_t object. + * + * \param[in] workers Maximum number of compressor threads, or 0 to compress + * inline (the default). Capped at the cores available to the process. + * + * \return #nmsg_res_success, or #nmsg_res_failure on an unbuffered output; + * #nmsg_res_memfail or #nmsg_res_failure if the pool cannot be built, and + * success without a pool on an output this does not apply to. Setting 0 + * returns any write error the pool had yet to report. + */ +nmsg_res +nmsg_output_set_zlib_workers(nmsg_output_t output, unsigned workers); + +/** + * Set when the compressor pool gives threads back. + * + * Work always goes to the lowest-numbered free compressor, so the rest of a + * pool that grew for a burst falls quiet and is culled. Write order is + * unaffected. May be called before or after nmsg_output_set_zlib_workers(). + * File outputs only. + * + * \param[in] output nmsg_output_t object. + * + * \param[in] min_workers Compressors culling leaves alone, 1 by default; 0 lets + * the pool empty. A floor on culling, not a number of threads to start. + * + * \param[in] idle_secs Idle seconds that cost a compressor its place, 300 by + * default. 0 disables culling. + */ +void +nmsg_output_set_zlib_cull(nmsg_output_t output, unsigned min_workers, + unsigned idle_secs); + #endif /* NMSG_OUTPUT_H */ diff --git a/nmsg/output_async.c b/nmsg/output_async.c new file mode 100644 index 000000000..1ccc9a42b --- /dev/null +++ b/nmsg/output_async.c @@ -0,0 +1,949 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The compressor pool for a stream output. The reorder buffer, its slot states + * and the threads that walk it are private here; output_nmsg.c submits sealed + * containers and supplies the compress and write callbacks. + */ + +/* Import. */ + +#include "private.h" + +#include "libmy/my_cpu.h" + +/* Data structures. */ + +/* + * A compressor pool: a ticket reorder buffer, up to nworkers compressor + * threads and one committer thread. + * + * Containers are ticketed under c_lock as they are sealed. Compression runs on + * whichever thread is free, but only the committer writes and only in ticket + * order, so the byte stream matches the synchronous path. + * + * A producer that finds every worker busy compresses inline rather than wait, + * so the pool is never slower than no pool. Workers spawn on demand, making + * 'nworkers' a ceiling rather than an allocation, and go away again once they + * have been idle long enough; see async_worker(). + */ + +/* + * Slots beyond one per worker, so a producer can run ahead of the compressors + * instead of waiting on its own ticket's slot. Headroom, not a requirement: + * depth == nworkers would still make progress. + */ +#define ASYNC_REORDER_MARGIN 8 + +/* Smallest reorder buffer worth allocating. */ +#define ASYNC_DEPTH_MIN 16 + +typedef enum { + slot_empty = 0, /* Free. */ + slot_work, /* Container waiting for a compressor. */ + slot_taken, /* A worker or a producer is compressing it. */ + slot_done, /* Compressed, waiting its turn to be written. */ + slot_frag /* Oversized: the committer must fragment it itself. */ +} async_slot_state; + +struct async_slot { + async_slot_state state; + nmsg_container_t co; /* slot_work, slot_frag */ + uint8_t *buf; /* slot_done */ + size_t buf_len; + nmsg_res res; +}; + +/* + * A compressor. The record outlives the thread: a culled worker leaves its + * 'tid' for the committer to join before the record is reused. Producers always + * take the lowest free index, so the front of the array carries the load; + * spread the work evenly instead and nothing is idle long enough to cull. + */ +struct async_worker { + pthread_t tid; + pthread_cond_t ready; /* Has a slot, or should look again. */ + struct async_slot *slot; /* Work handed over, or NULL. */ + bool idle; /* Waiting for work. */ + bool joinable; /* tid is valid and unjoined. */ + bool exited; /* Thread returned; needs a join. */ + struct nmsg_ostr_async *pool; +}; + +struct nmsg_ostr_async { + pthread_mutex_t lock; + pthread_cond_t commit_ready; /* The committer's slot is ready. */ + pthread_cond_t slot_free; /* A slot became slot_empty. */ + /* + * The ticket reorder buffer. Slot i serves every ticket with (ticket % + * depth) == i, so a producer runs at most 'depth' tickets ahead of + * commit_next and waits only for ticket T - depth to be written. + */ + struct async_slot *slots; + unsigned depth; + unsigned nworkers; /* Ceiling; workers start on demand. */ + unsigned nlive; /* Workers running now. */ + unsigned npeak; /* Most that ran at once. */ + unsigned min_workers; /* Workers culling leaves alone. */ + unsigned cull_secs; /* Idle seconds before a cull; 0 is off. */ + clockid_t cull_clock; /* The clock 'ready' was built with. */ + struct async_worker *workers; + uint64_t issued; /* Highest ticket claimed, plus one. */ + uint64_t commit_next; /* Ticket allowed to write now. */ + bool shutdown; + bool started; /* Committer exists; must be joined. */ + bool failed; /* No committer; stay inline. */ + bool spawn_failed; /* Worker spawn failed; stop trying. */ + pthread_t committer; + nmsg_res first_error; /* Sticky; surfaced by flush. */ + nmsg_output_t output; + uint64_t n_inline; /* Containers a producer compressed. */ + uint64_t n_waited; /* Producers that waited for a slot. */ + uint64_t n_culled; /* Workers that gave up their place. */ +}; + +/* + * Take a reference to the stream's pool, or return NULL if there is none or it + * is being torn down. Caller holds c_lock. + */ +struct nmsg_ostr_async * +_output_async_ref(struct nmsg_stream_output *ostr) +{ + if (ostr->so_pool == NULL || ostr->so_pool_closing) + return (NULL); + + ostr->so_inflight++; + + return (ostr->so_pool); +} + +/* Drop a reference taken by _output_async_ref() and wake any waiting teardown. */ +void _output_async_unref(struct nmsg_stream_output *ostr) +{ + pthread_mutex_lock(&ostr->c_lock); + assert(ostr->so_inflight > 0); + if (--ostr->so_inflight == 0) + pthread_cond_broadcast(&ostr->c_drained); + pthread_mutex_unlock(&ostr->c_lock); +} + +/* + * The compressor to hand the next container to, or NULL if all of them are + * busy. Lowest free index every time, so the front of the array takes the load + * and the back stays idle long enough to be worth culling. + * + * Taken in the same lock hold that hands it a slot, so a producer can never + * pick one that is on its way out. Caller holds pool->lock. + */ +static struct async_worker * +async_idle_take(struct nmsg_ostr_async *pool) +{ + unsigned i; + + for (i = 0; i < pool->nworkers; i++) { + if (pool->workers[i].idle) { + pool->workers[i].idle = false; + return (&pool->workers[i]); + } + } + + return (NULL); +} + +/* + * When a worker idle from now has outstayed its welcome. Read from the clock + * its condvar was built with. + */ +static bool +async_cull_deadline(const struct nmsg_ostr_async *pool, struct timespec *deadline) +{ + if (clock_gettime(pool->cull_clock, deadline) != 0) + return (false); + + deadline->tv_sec += pool->cull_secs; + + return (true); +} + +/* + * The floor a pool of this size can honour. At the ceiling there is nothing + * left to cull, so say so rather than quietly do nothing. + */ +static unsigned +async_min_workers_for(unsigned nworkers, unsigned min_workers) +{ + if (min_workers > nworkers) { + _nmsg_dprintf(2, "%s: floor of %u lowered to the %u compressor(s) " + "this output may run\n", + __func__, min_workers, nworkers); + min_workers = nworkers; + } + + return (min_workers); +} + +/* + * How many slots a pool of this size needs. + * + * One slot per worker covers everything that can be in flight, and the margin + * leaves room to deposit into. Past that, more depth only defers backpressure, + * and what it would be covering for is a slow write -- which is single-threaded + * on the committer, and no amount of depth helps. + * + * Sized here rather than fixed because a slot pins a compressed container until + * every earlier ticket is written: at the 1 MiB containers a file output uses, + * a buffer sized for the largest box is tens of megabytes that a two-worker + * output never needs. + */ +static unsigned +async_depth_for(unsigned nworkers) +{ + unsigned depth = nworkers + ASYNC_REORDER_MARGIN; + + return (depth < ASYNC_DEPTH_MIN ? ASYNC_DEPTH_MIN : depth); +} + +/* + * Compressor threads one output may have. Compression is CPU-bound, so more + * than one thread per core cannot help; past that they only add context + * switches and reorder buffer. + * + * This is a backstop for callers passing an arbitrary count. nmsgtool sizes + * its own request against the readers it is actually running. + */ +static unsigned +async_max_workers(void) +{ + return ((unsigned)my_ncpu()); +} + +/* Apply a cull policy to a running pool. */ +void _output_async_set_cull(struct nmsg_ostr_async *pool, unsigned min_workers, + unsigned idle_secs) +{ + unsigned i; + + pthread_mutex_lock(&pool->lock); + + pool->min_workers = async_min_workers_for(pool->nworkers, min_workers); + pool->cull_secs = idle_secs; + + /* Parked workers are waiting on the policy that has just been replaced. */ + for (i = 0; i < pool->nworkers; i++) + pthread_cond_signal(&pool->workers[i].ready); + + pthread_mutex_unlock(&pool->lock); +} + +/* Worker counts, for tests and diagnostics. Any of the outputs may be NULL. */ +void _output_async_counts(struct nmsg_ostr_async *pool, unsigned *live, + unsigned *peak, uint64_t *culled) +{ + pthread_mutex_lock(&pool->lock); + if (live != NULL) + *live = pool->nlive; + if (peak != NULL) + *peak = pool->npeak; + if (culled != NULL) + *culled = pool->n_culled; + pthread_mutex_unlock(&pool->lock); +} + +nmsg_res +_output_async_init(nmsg_output_t output, unsigned nworkers) +{ + struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool, *old; + nmsg_res res, old_res = nmsg_res_success; + unsigned depth, max_workers, zmin, zcull; + unsigned i, nconds = 0; + bool same_ceiling; + pthread_condattr_t cattr; + pthread_condattr_t *cattrp = NULL; + clockid_t cull_clock = CLOCK_REALTIME; + + if (nworkers == 0) + return (nmsg_res_success); + + max_workers = async_max_workers(); + if (nworkers > max_workers) + nworkers = max_workers; + + depth = async_depth_for(nworkers); + + /* + * The policy is set before the pool exists and carried across a + * replacement. Read with the pool under one c_lock hold, and the pool + * referenced, so a teardown cannot free it underneath. + */ + pthread_mutex_lock(&ostr->c_lock); + zmin = ostr->so_zmin; + zcull = ostr->so_zcull; + old = _output_async_ref(ostr); + same_ceiling = old != NULL && old->nworkers == nworkers; + pthread_mutex_unlock(&ostr->c_lock); + + /* + * A pool's ceiling cannot change in place, so a different count means + * a replacement. Build it before tearing the old one down, so a failed + * allocation leaves the working pool in place. + */ + if (same_ceiling) { + /* Nothing to rebuild, but the cull policy may have moved on. */ + _output_async_set_cull(old, zmin, zcull); + _output_async_unref(ostr); + return (nmsg_res_success); + } + + if (old != NULL) + _output_async_unref(ostr); + + pool = calloc(1, sizeof(*pool)); + if (pool == NULL) + return (nmsg_res_memfail); + + res = nmsg_res_memfail; + + pool->slots = calloc(depth, sizeof(*pool->slots)); + if (pool->slots == NULL) + goto fail_slots; + + pool->workers = calloc(nworkers, sizeof(*pool->workers)); + if (pool->workers == NULL) + goto fail_workers; + + res = nmsg_res_failure; + + if (pthread_mutex_init(&pool->lock, NULL) != 0) + goto fail_mutex; + if (pthread_cond_init(&pool->commit_ready, NULL) != 0) + goto fail_commit_ready; + if (pthread_cond_init(&pool->slot_free, NULL) != 0) + goto fail_slot_free; + + /* + * One attribute for every worker condvar, so any of them can carry a + * cull deadline. Monotonic, or a stepped wall clock retimes culling. + */ +#ifdef HAVE_PTHREAD_CONDATTR_SETCLOCK + if (pthread_condattr_init(&cattr) == 0) { + if (pthread_condattr_setclock(&cattr, CLOCK_MONOTONIC) == 0) { + cattrp = &cattr; + cull_clock = CLOCK_MONOTONIC; + } else { + pthread_condattr_destroy(&cattr); + } + } +#endif /* HAVE_PTHREAD_CONDATTR_SETCLOCK */ + + for (nconds = 0; nconds < nworkers; nconds++) { + if (pthread_cond_init(&pool->workers[nconds].ready, cattrp) != 0) + break; + pool->workers[nconds].pool = pool; + } + + if (cattrp != NULL) + pthread_condattr_destroy(cattrp); + + if (nconds < nworkers) + goto fail_worker_conds; + + pool->depth = depth; + pool->nworkers = nworkers; + pool->cull_clock = cull_clock; + pool->output = output; + + /* A no-op under c_lock if there is nothing to replace. */ + old_res = _output_async_destroy(output); + + pthread_mutex_lock(&ostr->c_lock); + + pool->min_workers = async_min_workers_for(nworkers, ostr->so_zmin); + pool->cull_secs = ostr->so_zcull; + + /* + * Tickets span the stream, not the pool, so a pool built mid-stream + * must start where the stream got to. + */ + pool->commit_next = pool->issued = ostr->so_ticket; + + /* Carry the old pool's unreported error rather than swallow it. */ + pool->first_error = old_res; + + ostr->so_pool = pool; + pthread_mutex_unlock(&ostr->c_lock); + + return (nmsg_res_success); + +fail_worker_conds: + for (i = 0; i < nconds; i++) + pthread_cond_destroy(&pool->workers[i].ready); + pthread_cond_destroy(&pool->slot_free); +fail_slot_free: + pthread_cond_destroy(&pool->commit_ready); +fail_commit_ready: + pthread_mutex_destroy(&pool->lock); +fail_mutex: + free(pool->workers); +fail_workers: + free(pool->slots); +fail_slots: + free(pool); + + /* Whatever pool was already in place is untouched and still running. */ + return (res); +} + +/* + * Stop the pool and reclaim it, writing everything still queued. Must run + * before the stream's fd, random and locks go away; the threads use all three. + */ +nmsg_res +_output_async_destroy(nmsg_output_t output) +{ + struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool; + nmsg_res res; + bool started; + unsigned i, npeak; + uint64_t n_culled, n_inline, n_waited; + + pthread_mutex_lock(&ostr->c_lock); + + /* + * One teardown at a time. A second waits the first out rather than + * returning, so a close cannot reach the fd while another thread's + * committer is still draining onto it. + */ + while (ostr->so_pool_closing) + pthread_cond_wait(&ostr->c_drained, &ostr->c_lock); + + pool = ostr->so_pool; + if (pool == NULL) { + pthread_mutex_unlock(&ostr->c_lock); + return (nmsg_res_success); + } + + /* + * Stop issuing tickets to the pool, then wait for the outstanding ones + * to arrive. Both under c_lock, which orders them against issuance: + * once this returns, no producer is still en route with a ticket. + */ + ostr->so_pool_closing = true; + while (ostr->so_inflight > 0) + pthread_cond_wait(&ostr->c_drained, &ostr->c_lock); + pthread_mutex_unlock(&ostr->c_lock); + + pthread_mutex_lock(&pool->lock); + pool->shutdown = true; /* Set under the lock: a thread about */ + started = pool->started; /* to wait would miss the wakeup. */ + npeak = pool->npeak; + n_culled = pool->n_culled; + n_inline = pool->n_inline; + n_waited = pool->n_waited; + for (i = 0; i < pool->nworkers; i++) + pthread_cond_signal(&pool->workers[i].ready); + pthread_cond_broadcast(&pool->commit_ready); + pthread_cond_broadcast(&pool->slot_free); + pthread_mutex_unlock(&pool->lock); + + /* + * The committer first: it is the only other thread that joins workers, + * so joining it settles 'joinable' before the loop reads it. No + * producer is left to spawn, and the committer already waits for every + * worker to deposit before it exits. + */ + if (started) + pthread_join(pool->committer, NULL); + for (i = 0; i < pool->nworkers; i++) { + if (pool->workers[i].joinable) + pthread_join(pool->workers[i].tid, NULL); + } + + if (n_inline > 0 || n_waited > 0 || n_culled > 0) + _nmsg_dprintf(2, "%s: %u of %u worker(s) at once, %u slot(s); " + "%" PRIu64 " container(s) compressed by the reader, " + "%" PRIu64 " wait(s) for a free slot, " + "%" PRIu64 " worker(s) culled\n", + __func__, npeak, pool->nworkers, pool->depth, + n_inline, n_waited, n_culled); + + /* Read after the joins; the threads write it until they exit. */ + res = pool->first_error; + + pthread_mutex_lock(&ostr->c_lock); + ostr->so_pool = NULL; + ostr->so_pool_closing = false; + pthread_cond_broadcast(&ostr->c_drained); /* Any teardown behind us. */ + pthread_mutex_unlock(&ostr->c_lock); + + for (i = 0; i < pool->nworkers; i++) + pthread_cond_destroy(&pool->workers[i].ready); + pthread_cond_destroy(&pool->slot_free); + pthread_cond_destroy(&pool->commit_ready); + pthread_mutex_destroy(&pool->lock); + free(pool->workers); + + /* + * Every slot is empty by now: producers are gone, the threads are + * joined, and the committer writes every ticket that was issued. Swept + * anyway, so the invariant is checked rather than assumed. + */ + for (i = 0; i < pool->depth; i++) { + assert(pool->slots[i].state == slot_empty); + nmsg_container_destroy(&pool->slots[i].co); + free(pool->slots[i].buf); + } + free(pool->slots); + free(pool); + + return (res); +} + +/* Note an error, keeping the first one seen. Caller holds pool->lock. */ +static void +async_record_error(struct nmsg_ostr_async *pool, nmsg_res res) +{ + if (res != nmsg_res_success && pool->first_error == nmsg_res_success) + pool->first_error = res; +} + +/* + * The slot this ticket owns is still held by the ticket 'depth' earlier. + * Caller holds pool->lock. + */ +static bool +async_slot_busy(const struct nmsg_ostr_async *pool, uint64_t ticket) +{ + return (ticket >= pool->commit_next + pool->depth); +} + +/* + * Every ticket the pool was given has been written. Caller holds pool->lock. + */ +static bool +async_all_written(const struct nmsg_ostr_async *pool) +{ + return (pool->commit_next >= pool->issued); +} + +/* + * Compressor thread. Waits to be handed a slot rather than looking for one, so + * the producer decides which worker runs and the rest go quiet. + * + * A worker idle for cull_secs gives up its place, down to min_workers. That + * decision and clearing its idle flag are one lock hold, so a producer can + * never hand work to a thread on its way out. + */ +static void * +async_worker(void *arg) +{ + struct async_worker *self = (struct async_worker *)arg; + struct nmsg_ostr_async *pool = self->pool; + bool timed_out = false; + + pthread_mutex_lock(&pool->lock); + + for (;;) { + struct async_slot *slot = self->slot; + nmsg_container_t co; + uint8_t *buf; + size_t buf_len; + nmsg_res res; + + if (slot == NULL) { + struct timespec deadline; + + /* + * Exit only once producers have stopped, so a + * container queued just before shutdown is still + * written. Happens on every clean SIGTERM. + */ + if (pool->shutdown) + break; + + if (timed_out && pool->cull_secs > 0 && + pool->nlive > pool->min_workers) { + pool->n_culled++; + break; + } + + timed_out = false; + + self->idle = true; + + /* + * No deadline with culling off, nor at the floor, + * where it could only cost wakeups. The floor is not + * tied to particular threads: grow again and the + * workers added on top are the ones that time out. + */ + bool park_indefinitely = pool->cull_secs == 0 || + pool->nlive <= pool->min_workers; + + if (park_indefinitely || + !async_cull_deadline(pool, &deadline)) { + pthread_cond_wait(&self->ready, &pool->lock); + } else { + int wait_res; + + wait_res = pthread_cond_timedwait(&self->ready, + &pool->lock, + &deadline); + timed_out = wait_res == ETIMEDOUT; + } + continue; + } + + self->slot = NULL; + slot->state = slot_taken; + co = slot->co; + slot->co = NULL; + pthread_mutex_unlock(&pool->lock); + + res = _output_nmsg_container_compress(pool->output, &co, &buf, &buf_len); + + pthread_mutex_lock(&pool->lock); + timed_out = false; + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); + } + + self->idle = false; + self->exited = true; + pool->nlive--; + + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * Join the workers culling has retired, freeing their records to be spawned + * into again. Runs on the committer because the join is unbounded -- the thread + * has returned but still has the C library's teardown to be scheduled for -- + * and the committer is the one thread that may block for it. + * + * Being the only reaper is what lets a producer skip a record on 'joinable' + * alone, so nothing can claim one while the lock is dropped here. + * Caller holds pool->lock. + */ +static void +async_reap_exited(struct nmsg_ostr_async *pool) +{ + unsigned i; + + for (i = 0; i < pool->nworkers; i++) { + struct async_worker *worker = &pool->workers[i]; + + if (!worker->exited) + continue; + + pthread_mutex_unlock(&pool->lock); + pthread_join(worker->tid, NULL); + pthread_mutex_lock(&pool->lock); + + /* Cleared last: until then the record is not free. */ + worker->exited = false; + worker->joinable = false; + } +} + +/* + * Takes tickets strictly in order, so the file matches what the synchronous + * path would have produced. The only writer while the pool is up: a producer + * writes inline only when there is no pool to take its ticket. + */ +static void * +async_committer(void *arg) +{ + struct nmsg_ostr_async *pool = (struct nmsg_ostr_async *)arg; + + pthread_mutex_lock(&pool->lock); + + for (;;) { + struct async_slot *slot = &pool->slots[pool->commit_next % pool->depth]; + uint8_t *buf; + size_t buf_len; + nmsg_container_t co; + nmsg_res res = nmsg_res_success; + + switch (slot->state) { + case slot_done: + buf = slot->buf; + buf_len = slot->buf_len; + res = slot->res; + slot->buf = NULL; + pthread_mutex_unlock(&pool->lock); + + /* + * _output_nmsg_send_buffer() frees buf; not called on + * a compression failure, where buf is NULL anyway. + */ + if (res == nmsg_res_success) + res = _output_nmsg_send_buffer(pool->output, buf, buf_len); + + pthread_mutex_lock(&pool->lock); + break; + + case slot_frag: + co = slot->co; + slot->co = NULL; + pthread_mutex_unlock(&pool->lock); + + /* Takes the container by value and destroys it. */ + res = _output_nmsg_frag_write(pool->output, co); + + pthread_mutex_lock(&pool->lock); + break; + + case slot_empty: + case slot_work: + case slot_taken: + /* + * Not ready. Exit only once producers have stopped and + * every ticket they issued has been written. + */ + if (pool->shutdown && async_all_written(pool)) + goto out; + pthread_cond_wait(&pool->commit_ready, &pool->lock); + continue; + } + + async_record_error(pool, res); + slot->state = slot_empty; + pool->commit_next++; + pthread_cond_broadcast(&pool->slot_free); + + async_reap_exited(pool); + } + +out: + pthread_mutex_unlock(&pool->lock); + + return (NULL); +} + +/* + * Start the committer, under pool->lock on first submit rather than at + * configure time: nmsgtool creates its outputs before daemonize(), which is a + * bare fork() no thread survives. Workers are added later by + * async_spawn_worker(). + */ +static void +async_start(struct nmsg_ostr_async *pool) +{ + int pthread_res; + + pthread_res = pthread_create(&pool->committer, NULL, async_committer, pool); + if (pthread_res != 0) { + /* + * Nothing can be written without a committer, so abandon the + * pool and compress inline. Nothing has been deposited yet to + * unwind, but a flush may already be waiting on a ticket that + * will now never be committed. + */ + pool->failed = true; + pthread_cond_broadcast(&pool->slot_free); + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", __func__, + strerror(pthread_res)); + return; + } + + pool->started = true; +} + +/* + * Add a compressor, up to the ceiling. Called under pool->lock when a producer + * finds no idle worker, so the pool grows to the load it sees. + * + * Takes only a free record: one a culled worker left stays off limits until the + * committer has joined it, so this never blocks the reader. Returns NULL if + * there is none or the spawn failed which is not fatal, the caller compresses that + * container itself. + */ +static struct async_worker * +async_spawn_worker(struct nmsg_ostr_async *pool) +{ + struct async_worker *worker = NULL; + unsigned i; + int pthread_res; + + if (pool->spawn_failed) + return (NULL); + + for (i = 0; i < pool->nworkers; i++) { + if (!pool->workers[i].joinable) { + worker = &pool->workers[i]; + break; + } + } + + if (worker == NULL) + return (NULL); + + assert(!worker->idle && worker->slot == NULL); + + pthread_res = pthread_create(&worker->tid, NULL, async_worker, worker); + if (pthread_res != 0) { + /* + * Latched, not retried: culling holds the pool below its + * ceiling, so retrying means this syscall per container. + */ + pool->spawn_failed = true; + _nmsg_dprintf(1, "%s: pthread_create() failed: %s\n", + __func__, strerror(pthread_res)); + return (NULL); + } + + worker->joinable = true; + pool->nlive++; + if (pool->nlive > pool->npeak) + pool->npeak = pool->nlive; + + return (worker); +} + +/* + * Wait until every ticket below 'upto' has been written, and take any error the + * pool recorded. + */ +nmsg_res +_output_async_drain(struct nmsg_ostr_async *pool, uint64_t upto) +{ + nmsg_res res; + + if (pool == NULL) + return (nmsg_res_success); + + pthread_mutex_lock(&pool->lock); + while (!pool->failed && pool->commit_next < upto) + pthread_cond_wait(&pool->slot_free, &pool->lock); + res = pool->first_error; + pool->first_error = nmsg_res_success; + pthread_mutex_unlock(&pool->lock); + + return (res); +} + +/* + * Hand a sealed container to the pool, consuming it only if the pool takes it. + * Returns false if it does not, leaving it for the caller to write. The pool + * reference is released either way. + * + * *res_out carries an EARLIER container's error; this one is not written yet. + */ +bool _output_async_submit(struct nmsg_ostr_async *pool, nmsg_output_t output, + nmsg_container_t *co, bool is_frag, uint64_t ticket, + nmsg_res *res_out) +{ + struct nmsg_stream_output *ostr = output->stream; + struct async_slot *slot; + struct async_worker *worker = NULL; + nmsg_res res = nmsg_res_success; + uint8_t *buf; + size_t buf_len; + bool inline_compress = false; + + pthread_mutex_lock(&pool->lock); + + /* + * No teardown can be running: it waits out the tickets already issued + * before it sets shutdown, and this producer is holding one. That is + * also what lets the slot wait below trust its slot is free. + */ + if (!pool->started && !pool->failed) + async_start(pool); + + /* Read after the attempt: async_start() sets started or failed. */ + if (!pool->started) { + pthread_mutex_unlock(&pool->lock); + _output_async_unref(ostr); + return (false); + } + + slot = &pool->slots[ticket % pool->depth]; + + /* + * Wait for the slot this ticket owns, released by the ticket 'depth' + * earlier. Only reached when the writer is a full 'depth' behind. + */ + if (async_slot_busy(pool, ticket)) { + pool->n_waited++; + while (async_slot_busy(pool, ticket)) + pthread_cond_wait(&pool->slot_free, &pool->lock); + } + + assert(slot->state == slot_empty); + + if (ticket >= pool->issued) + pool->issued = ticket + 1; + + if (is_frag) { + /* Only the committer fragments; see async_committer(). */ + slot->co = *co; + *co = NULL; + slot->state = slot_frag; + pthread_cond_broadcast(&pool->commit_ready); + } else { + /* An idle compressor, or a new one if the pool may still grow. */ + worker = async_idle_take(pool); + if (worker == NULL) + worker = async_spawn_worker(pool); + + if (worker != NULL) { + /* Hand the container over and get back to reading. */ + slot->co = *co; + *co = NULL; + slot->state = slot_work; + worker->slot = slot; + pthread_cond_signal(&worker->ready); + } else { + /* Everyone busy and at the ceiling: compress here. */ + slot->state = slot_taken; + pool->n_inline++; + inline_compress = true; + } + } + + /* + * Reported but not consumed: a write that never reached disk must still + * be there for the flush or close to find. Cleared by the drain. + */ + res = pool->first_error; + pthread_mutex_unlock(&pool->lock); + + if (inline_compress) { + nmsg_res c_res = _output_nmsg_container_compress(output, co, &buf, &buf_len); + + pthread_mutex_lock(&pool->lock); + slot->buf = buf; + slot->buf_len = buf_len; + slot->res = c_res; + slot->state = slot_done; + pthread_cond_broadcast(&pool->commit_ready); + pthread_mutex_unlock(&pool->lock); + } + + *res_out = res; + _output_async_unref(ostr); + + return (true); +} diff --git a/nmsg/output_nmsg.c b/nmsg/output_nmsg.c index f97c7b855..2d911838e 100644 --- a/nmsg/output_nmsg.c +++ b/nmsg/output_nmsg.c @@ -21,31 +21,64 @@ /* Forward. */ static nmsg_res container_write(nmsg_output_t, nmsg_container_t*); -static nmsg_res frag_write(nmsg_output_t, nmsg_container_t); -static nmsg_res send_buffer(nmsg_output_t, uint8_t *buf, size_t len); +static nmsg_res container_submit(nmsg_output_t, nmsg_container_t *, bool, uint64_t, + struct nmsg_ostr_async *); /* Internal functions. */ nmsg_res _output_nmsg_flush(nmsg_output_t output) { + struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool, *submit_pool = NULL; nmsg_res res = nmsg_res_success; + nmsg_res drain_res; + nmsg_container_t old_c = NULL; + uint64_t ticket = 0, upto; - pthread_mutex_lock(&output->stream->c_lock); + pthread_mutex_lock(&ostr->c_lock); - if (nmsg_container_get_num_payloads(output->stream->c) > 0) { + /* + * The pool is picked up here rather than read again later, so a teardown + * running alongside this flush cannot free it underneath. + */ + pool = _output_async_ref(ostr); - /* Process container; container is destroyed. */ - res = container_write(output, &output->stream->c); + if (nmsg_container_get_num_payloads(ostr->c) > 0) { + old_c = ostr->c; + ticket = ostr->so_ticket++; + submit_pool = _output_async_ref(ostr); - output->stream->c = nmsg_container_init(output->stream->bufsz); - if (output->stream->c == NULL) + ostr->c = nmsg_container_init(ostr->bufsz); + if (ostr->c == NULL) res = nmsg_res_memfail; else - nmsg_container_set_sequence(output->stream->c, output->stream->do_sequence); + nmsg_container_set_sequence(ostr->c, ostr->do_sequence); + } + + /* Everything sealed so far, including the container just taken. */ + upto = ostr->so_ticket; + + pthread_mutex_unlock(&ostr->c_lock); + /* + * Submitted outside c_lock: a submit can wait for a free slot, and + * holding c_lock across that would stall every reader thread. + */ + if (old_c != NULL) { + nmsg_res sub_res; + + sub_res = container_submit(output, &old_c, false, ticket, submit_pool); + if (res == nmsg_res_success) + res = sub_res; } - pthread_mutex_unlock(&output->stream->c_lock); + /* A flush means written, so wait out anything the pool still holds. */ + drain_res = _output_async_drain(pool, upto); + if (res == nmsg_res_success) + res = drain_res; + + if (pool != NULL) + _output_async_unref(ostr); return (res); } @@ -54,8 +87,10 @@ nmsg_res _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { Nmsg__NmsgPayload *np; struct nmsg_stream_output *ostr = output->stream; + struct nmsg_ostr_async *pool; nmsg_container_t old_c, new_c; - nmsg_res res; + nmsg_res res, sub_res, pending = nmsg_res_success; + uint64_t ticket; bool must_flush, is_buffered; assert(msg->np != NULL); @@ -78,6 +113,8 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { retry: must_flush = false; old_c = new_c = NULL; + pool = NULL; + ticket = 0; pthread_mutex_lock(&ostr->c_lock); /* Lock for add to container. */ @@ -96,11 +133,11 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { * container for the other threads to use. */ is_buffered = ostr->buffered; /* Save this value. */ - if ((res == nmsg_res_container_full) || - (res == nmsg_res_success && is_buffered == false) || - (res == nmsg_res_container_overfull)) { - must_flush = true; /* Will flush container below. */ + must_flush = (res == nmsg_res_container_full) || + (res == nmsg_res_success && is_buffered == false) || + (res == nmsg_res_container_overfull); + if (must_flush) { /* Create replacement container. */ new_c = nmsg_container_init(ostr->bufsz); if (new_c == NULL) { @@ -112,32 +149,94 @@ _output_nmsg_write(nmsg_output_t output, nmsg_message_t msg) { old_c = ostr->c; /* Process old, proceed with new. */ ostr->c = new_c; + + /* + * Ticket taken under c_lock, where the container's contents + * become final. Taking it after the unlock would order + * containers by which thread won the race, not by fill order. + * The pool is claimed in the same hold, so ticket and pool are + * chosen together. + */ + ticket = ostr->so_ticket++; + pool = _output_async_ref(ostr); } pthread_mutex_unlock(&ostr->c_lock); /* Release locked container to other threads. */ if (!must_flush) /* Nothing more to do here. */ - return (res); + goto out; /* Reaching here WILL flush the prior container. */ if (res == nmsg_res_container_full) { /* Doesn't include current message. */ - res = container_write(output, &old_c); /* Write data from prior container. */ - if (res != nmsg_res_success) - return (res); + /* Write data from prior container. */ + sub_res = container_submit(output, &old_c, false, ticket, pool); + + /* + * Kept rather than returned: with a pool the result belongs to + * some earlier container, and returning here would drop this + * message. + */ + if (pending == nmsg_res_success) + pending = sub_res; /* Proceed to write current message to new container. */ goto retry; } else if (res == nmsg_res_success && is_buffered == false) { /* Includes current message. */ - res = container_write(output, &old_c); + res = container_submit(output, &old_c, false, ticket, pool); } else if (res == nmsg_res_container_overfull) { /* Includes current message. */ - res = frag_write(output, old_c); + res = container_submit(output, &old_c, true, ticket, pool); } - return (res); +out: + /* An earlier container's error outranks this one's disposition. */ + return (pending != nmsg_res_success ? pending : res); } /* Private functions. */ +/* + * Compress one container into a standalone buffer. The container is destroyed + * either way, as container_write() does. + */ +nmsg_res +_output_nmsg_container_compress(nmsg_output_t output, nmsg_container_t *co, + uint8_t **buf, size_t *buf_len) +{ + struct nmsg_stream_output *ostr = output->stream; + nmsg_res res; + uint32_t seq; + uint8_t *shrunk; + + /* + * Multiple threads can enter here at once, so numbers go out in + * compression order, not write order. Safe only because pools are + * file-only and file outputs leave do_sequence false; a pool on a + * sequenced output would scramble them. + */ + seq = atomic_fetch_add_explicit(&ostr->so_sequence_num, 1, memory_order_relaxed); + + res = nmsg_container_serialize(*co, buf, buf_len, true, /* do_header */ + ostr->do_zlib, seq, ostr->sequence_id); + nmsg_container_destroy(co); + + if (res != nmsg_res_success) { + *buf = NULL; + *buf_len = 0; + return (res); + } + + /* + * serialize() allocates for the worst case, twice the unpacked + * estimate. A slot pins that until every earlier ticket is written, so + * hand it back. + */ + shrunk = realloc(*buf, *buf_len); + if (shrunk != NULL) + *buf = shrunk; + + return (res); +} + /* * Send/write the contents of a container. * Container is destroyed, whether contents are successfully processed or not. @@ -147,23 +246,49 @@ container_write(nmsg_output_t output, nmsg_container_t *co) { nmsg_res res; size_t buf_len; - uint32_t seq; uint8_t *buf; - /* Multiple threads can enter here at once. */ - seq = atomic_fetch_add_explicit(&output->stream->so_sequence_num, 1, memory_order_relaxed); + res = _output_nmsg_container_compress(output, co, &buf, &buf_len); + if (res != nmsg_res_success) + return (res); - res = nmsg_container_serialize(*co, &buf, &buf_len, true, /* do_header */ - output->stream->do_zlib, seq, output->stream->sequence_id); + return (_output_nmsg_send_buffer(output, buf, buf_len)); +} - if (res != nmsg_res_success) - goto out; +/* Compress and write on the calling thread. The container is consumed. */ +static nmsg_res +container_submit_inline(nmsg_output_t output, nmsg_container_t *co, bool is_frag) +{ + if (is_frag) { + nmsg_container_t tmp = *co; - res = send_buffer(output, buf, buf_len); -out: - nmsg_container_destroy(co); + /* Takes the container by value and destroys it. */ + *co = NULL; + return (_output_nmsg_frag_write(output, tmp)); + } - return (res); + return (container_write(output, co)); +} + +/* + * Hand a finished container to the pool, or process it inline if there is no + * pool or it declines. The container is consumed either way. 'pool' is the + * reference taken when the ticket was issued, so it is NULL exactly when the + * ticket was never promised to a pool. + */ +static nmsg_res +container_submit(nmsg_output_t output, nmsg_container_t *co, bool is_frag, + uint64_t ticket, struct nmsg_ostr_async *pool) +{ + nmsg_res res; + bool taken_by_pool; + + taken_by_pool = pool != NULL && + _output_async_submit(pool, output, co, is_frag, ticket, &res); + if (taken_by_pool) + return (res); + + return (container_submit_inline(output, co, is_frag)); } static nmsg_res @@ -256,8 +381,8 @@ write_file(int fd, uint8_t *buf, size_t len) * * Returns status of send. */ -static nmsg_res -send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) +nmsg_res +_output_nmsg_send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) { struct nmsg_stream_output *ostr = output->stream; nmsg_res res; @@ -293,6 +418,52 @@ send_buffer(nmsg_output_t output, uint8_t *buf, size_t len) return (res); } +#ifdef HAVE_LIBRDKAFKA +nmsg_res +_output_kafka_payload_write(nmsg_output_t output, nmsg_message_t msg) { + nmsg_res res; + struct nmsg_strbuf_storage key_sbs; + struct nmsg_strbuf *key_sb = NULL; + uint8_t *buf = NULL, *key = NULL; + size_t buf_len, key_len = 0; + + assert(msg->np != NULL); + + buf_len = nmsg__nmsg_payload__get_packed_size(msg->np); + buf = malloc(buf_len); + if (buf == NULL) + return nmsg_res_memfail; + nmsg__nmsg_payload__pack(msg->np, buf); + + if (output->kafka->key_field != NULL) { + key_sb = _nmsg_strbuf_init(&key_sbs); + res = _nmsg_message_get_field_value_as_key(msg, output->kafka->key_field, key_sb); + if (res != nmsg_res_success) + goto out; + + key_len = nmsg_strbuf_len(key_sb); + key = (uint8_t *) key_sb->data; + } + + /* kafka_write() takes ownership of buf */ + res = kafka_write(output->kafka->ctx, key, key_len, buf, buf_len); + buf = NULL; + +out: + if (buf != NULL) + free(buf); + if (key_sb != NULL) + _nmsg_strbuf_destroy(&key_sbs); + return res; +} + +nmsg_res +_output_kafka_payload_flush(nmsg_output_t output) { + kafka_flush(output->kafka->ctx); + return nmsg_res_success; +} +#endif /* HAVE_LIBRDKAFKA */ + static void header_serialize(uint8_t *buf, uint8_t flags, uint32_t len) { @@ -309,8 +480,8 @@ header_serialize(uint8_t *buf, uint8_t flags, uint32_t len) store_net32(buf, len); } -static nmsg_res -frag_write(nmsg_output_t output, nmsg_container_t co) +nmsg_res +_output_nmsg_frag_write(nmsg_output_t output, nmsg_container_t co) { Nmsg__NmsgFragment nf; struct nmsg_stream_output *ostr = output->stream; @@ -346,7 +517,7 @@ frag_write(nmsg_output_t output, nmsg_container_t co) if (ostr->do_zlib && len <= max_fragsz) { /* write out the unfragmented NMSG container */ - res = send_buffer(output, packed, len); + res = _output_nmsg_send_buffer(output, packed, len); goto frag_out; } @@ -380,7 +551,7 @@ frag_write(nmsg_output_t output, nmsg_container_t co) fraglen += NMSG_HDRLSZ_V2; /* send the serialized fragment */ - res = send_buffer(output, frag_packed, fraglen); + res = _output_nmsg_send_buffer(output, frag_packed, fraglen); } free(packed); diff --git a/nmsg/pcap_input.c b/nmsg/pcap_input.c index d7bba7e06..61e114cc0 100644 --- a/nmsg/pcap_input.c +++ b/nmsg/pcap_input.c @@ -160,6 +160,10 @@ nmsg_pcap_input_setfilter_raw(nmsg_pcap_t pcap, const char *userbpft) { return (nmsg_res_failure); } pcap->userbpft = strdup(userbpft); + if (pcap->userbpft == NULL) { + pcap_freecode(&pcap->userbpf); + return (nmsg_res_memfail); + } /* test if we can skip vlan tags */ res = pcap_compile(pcap->handle, &bpf, "vlan and ip", 1, 0); @@ -241,6 +245,10 @@ nmsg_pcap_input_setfilter(nmsg_pcap_t pcap, const char *userbpft) { return (nmsg_res_failure); } pcap->userbpft = strdup(userbpft); + if (pcap->userbpft == NULL) { + pcap_freecode(&pcap->userbpf); + return (nmsg_res_memfail); + } /* test if we can skip ip6 */ res = nmsg_asprintf(&tmp, "(%s) and %s", userbpft, bpf_ip6); diff --git a/nmsg/private.h b/nmsg/private.h index bcff5b574..37947776e 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -66,9 +66,7 @@ #include #endif /* HAVE_LIBRDKAFKA */ -#ifdef HAVE_JSON_C #include -#endif /* HAVE_JSON_C */ #include "nmsg.h" #include "nmsg.pb-c.h" @@ -253,8 +251,6 @@ struct nmsg_pres { /* nmsg_json: used by nmsg_input and nmsg_output */ struct nmsg_json { -#ifdef HAVE_JSON_C -#endif /* HAVE_JSON_C */ pthread_mutex_t lock; FILE *fp; int orig_fd; @@ -311,6 +307,19 @@ struct nmsg_stream_input { nmsg_input_stream_read_fp stream_read_fp; }; +/* + * Asynchronous compressor for a stream output. + */ +struct nmsg_ostr_async; + +/* + * Compressor threads left alone when the pool shrinks, and the idle time that + * costs a thread its place. Defaults rather than constants: nmsg_output_set_zlib_cull() + * overrides both, and output_open_stream_base() seeds every stream with them. + */ +#define NMSG_ZCULL_MIN_WORKERS_DEFAULT 1 +#define NMSG_ZCULL_SECS_DEFAULT 300 + /* nmsg_stream_output: used by nmsg_output */ struct nmsg_stream_output { pthread_mutex_t c_lock; /* Container lock. */ @@ -335,6 +344,14 @@ struct nmsg_stream_output { bool do_sequence; atomic_uint_fast32_t so_sequence_num; uint64_t sequence_id; + /* The five below are guarded by c_lock. */ + uint64_t so_ticket; /* Next container ticket. */ + unsigned so_inflight; /* Tickets owed to the pool. */ + bool so_pool_closing;/* Teardown started. */ + unsigned so_zmin; /* Compressors culling leaves. */ + unsigned so_zcull; /* Idle seconds before a cull. */ + pthread_cond_t c_drained; /* so_inflight == 0. */ + struct nmsg_ostr_async *so_pool; /* Async compressor, or NULL. */ }; /* nmsg_callback_output: used by nmsg_output */ @@ -547,6 +564,7 @@ nmsg_res _input_nmsg_read_container_file(nmsg_input_t, Nmsg__Nmsg **); nmsg_res _input_nmsg_read_container_sock(nmsg_input_t, Nmsg__Nmsg **); #ifdef HAVE_LIBRDKAFKA nmsg_res _input_nmsg_read_container_kafka(nmsg_input_t, Nmsg__Nmsg **); +nmsg_res _input_kafka_payload_read(nmsg_input_t, nmsg_message_t *); #endif /* HAVE_LIBRDKAFKA */ #ifdef HAVE_LIBZMQ nmsg_res _input_nmsg_read_container_zmq(nmsg_input_t, Nmsg__Nmsg **); @@ -589,6 +607,26 @@ nmsg_output_t _output_open_kafka(void *s, size_t bufsz); /* from output_nmsg.c */ nmsg_res _output_nmsg_flush(nmsg_output_t); nmsg_res _output_nmsg_write(nmsg_output_t, nmsg_message_t); +nmsg_res _output_nmsg_container_compress(nmsg_output_t, nmsg_container_t *, + uint8_t **, size_t *); +nmsg_res _output_nmsg_send_buffer(nmsg_output_t, uint8_t *, size_t); +nmsg_res _output_nmsg_frag_write(nmsg_output_t, nmsg_container_t); + +/* from output_async.c */ +nmsg_res _output_async_init(nmsg_output_t, unsigned); +nmsg_res _output_async_destroy(nmsg_output_t); +void _output_async_set_cull(struct nmsg_ostr_async *, unsigned, unsigned); +void _output_async_counts(struct nmsg_ostr_async *, unsigned *, + unsigned *, uint64_t *); +nmsg_res _output_async_drain(struct nmsg_ostr_async *, uint64_t); +struct nmsg_ostr_async *_output_async_ref(struct nmsg_stream_output *); +void _output_async_unref(struct nmsg_stream_output *); +bool _output_async_submit(struct nmsg_ostr_async *, nmsg_output_t, + nmsg_container_t *, bool, uint64_t, nmsg_res *); +#ifdef HAVE_LIBRDKAFKA +nmsg_res _output_kafka_payload_write(nmsg_output_t, nmsg_message_t); +nmsg_res _output_kafka_payload_flush(nmsg_output_t); +#endif /* HAVE_LIBRDKAFKA */ /* from output_pres.c */ nmsg_res _output_pres_write(nmsg_output_t, nmsg_message_t); diff --git a/nmsg/strbuf.c b/nmsg/strbuf.c index 2749bba55..ece4f644d 100644 --- a/nmsg/strbuf.c +++ b/nmsg/strbuf.c @@ -69,10 +69,9 @@ _nmsg_strbuf_expand(struct nmsg_strbuf *sb, size_t len) { ssize_t avail = _nmsg_strbuf_avail(sb); assert(avail >= 0); - /* increase buffer size if necessary */ if (needed > avail) { size_t offset = sb->pos - sb->data; - ssize_t new_bufsz = 2 * sb->bufsz; + ssize_t new_bufsz = 2 * (sb->bufsz == 0 ? 1 : sb->bufsz); void *ptr; while (new_bufsz - (ssize_t) sb->bufsz < needed) { diff --git a/nmsg/zbuf.c b/nmsg/zbuf.c index 4af73dbd5..713eea3fd 100644 --- a/nmsg/zbuf.c +++ b/nmsg/zbuf.c @@ -126,6 +126,9 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, uint32_t my_ulen; load_net32(z_buf, &my_ulen); + if (my_ulen > NMSG_WBUFSZ_MAX) + return (nmsg_res_memfail); + z_buf += 4; *u_len = my_ulen; @@ -133,7 +136,7 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, if (*u_buf == NULL) return (nmsg_res_memfail); - zb->zs.avail_in = z_len; + zb->zs.avail_in = z_len - 4; zb->zs.next_in = z_buf; zb->zs.avail_out = *u_len; zb->zs.next_out = *u_buf; @@ -142,6 +145,7 @@ nmsg_zbuf_inflate(nmsg_zbuf_t zb, size_t z_len, u_char *z_buf, if (zret != Z_STREAM_END || zb->zs.avail_out != 0) { _nmsg_dprintf(1, "%s: inflate() failed\n", __func__); free(*u_buf); + *u_buf = NULL; return (nmsg_res_failure); } zret = inflateReset(&zb->zs); diff --git a/src/io.c b/src/io.c index 2b11cc453..6b9748fb2 100644 --- a/src/io.c +++ b/src/io.c @@ -209,7 +209,7 @@ add_sock_output(nmsgtool_ctx *c, const char *ss) { } } -#if (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) +#if (defined HAVE_LIBRDKAFKA) static void _add_kafka_json_input(nmsgtool_ctx *c, const char *str_address) { nmsg_input_t input; @@ -233,15 +233,15 @@ _add_kafka_json_input(nmsgtool_ctx *c, const char *str_address) { str_address); c->n_inputs += 1; } -#else /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#else /* (defined HAVE_LIBRDKAFKA) */ static void _add_kafka_json_input(nmsgtool_ctx *c __attribute__((unused)), const char *str_address __attribute__((unused))) { - fprintf(stderr, "%s: Error: compiled without librdkafka or json-c support\n", + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", argv_program); exit(EXIT_FAILURE); } -#endif /* (defined HAVE_JSON_C) && (defined HAVE_LIBRDKAFKA) */ +#endif /* (defined HAVE_LIBRDKAFKA) */ #ifdef HAVE_LIBRDKAFKA static void @@ -273,7 +273,74 @@ _add_kafka_json_output(nmsgtool_ctx *c, const char *str_address) { static void _add_kafka_json_output(nmsgtool_ctx *c __attribute__((unused)), const char *str_address __attribute__((unused))) { - fprintf(stderr, "%s: Error: compiled without librdkafka or json-c support\n", + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_LIBRDKAFKA */ + +#ifdef HAVE_LIBRDKAFKA +static void +_add_kafka_payload_output(nmsgtool_ctx *c, const char *str_address) { + nmsg_res res; + nmsg_output_t output; + + output = nmsg_output_open_kafka_payload(str_address, c->kafka_key_field); + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg Kafka nmsgp output: %s\n", argv_program, str_address); + if (output == NULL) { + fprintf(stderr, "%s: nmsg_output_open_kafka_payload() failed\n", argv_program); + exit(1); + } + setup_nmsg_output(c, output); + if (c->kicker != NULL) + res = nmsg_io_add_output(c->io, output, (void *) -1); + else + res = nmsg_io_add_output(c->io, output, NULL); + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_output() failed\n", argv_program); + exit(1); + } + c->n_outputs += 1; +} +#else /* HAVE_LIBRDKAFKA */ +static void +_add_kafka_payload_output(nmsgtool_ctx *c __attribute__((unused)), + const char *str_address __attribute__((unused))) +{ + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_LIBRDKAFKA */ + +#ifdef HAVE_LIBRDKAFKA +static void +_add_kafka_payload_input(nmsgtool_ctx *c, const char *str_address) { + nmsg_res res; + nmsg_input_t input; + + input = nmsg_input_open_kafka_payload(str_address); + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg Kafka nmsgp input: %s\n", argv_program, str_address); + if (input == NULL) { + fprintf(stderr, "%s: nmsg_input_open_kafka_payload() failed\n", argv_program); + exit(1); + } + setup_nmsg_input(c, input); + res = nmsg_io_add_input(c->io, input, NULL); + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_input() failed\n", argv_program); + exit(1); + } + c->n_inputs += 1; +} +#else /* HAVE_LIBRDKAFKA */ +static void +_add_kafka_payload_input(nmsgtool_ctx *c __attribute__((unused)), + const char *str_address __attribute__((unused))) +{ + fprintf(stderr, "%s: Error: compiled without librdkafka support\n", argv_program); exit(EXIT_FAILURE); } @@ -351,18 +418,18 @@ add_kafka_input(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_input(c, addr); return; } -#ifdef HAVE_JSON_C + addr = _strip_prefix_if_exists(str_address, "nmsgp:"); + if (addr != NULL) { + _add_kafka_payload_input(c, addr); + return; + } addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_input(c, addr); return; } - fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", - argv_program); -#else /* HAVE_JSON_C */ - fprintf(stderr, "%s: Error: nmsg protocol must be set for Kafka topic\n", + fprintf(stderr, "%s: Error: nmsg, nmsgp, or json protocol must be set for Kafka endpoint\n", argv_program); -#endif /* HAVE_JSON_C */ exit(EXIT_FAILURE); } @@ -373,18 +440,18 @@ add_kafka_output(nmsgtool_ctx *c, const char *str_address) { _add_kafka_nmsg_output(c, addr); return; } -#ifdef HAVE_JSON_C + addr = _strip_prefix_if_exists(str_address, "nmsgp:"); + if (addr != NULL) { + _add_kafka_payload_output(c, addr); + return; + } addr = _strip_prefix_if_exists(str_address, "json:"); if (addr != NULL) { _add_kafka_json_output(c, addr); return; } - fprintf(stderr, "%s: Error: nmsg or json protocol must be set for Kafka topic\n", - argv_program); -#else /* HAVE_JSON_C */ - fprintf(stderr, "%s: Error: nmsg protocol must be set for Kafka topic\n", + fprintf(stderr, "%s: Error: nmsg, nmsgp, or json protocol must be set for Kafka endpoint\n", argv_program); -#endif /* HAVE_JSON_C */ exit(EXIT_FAILURE); } @@ -531,6 +598,7 @@ add_file_output(nmsgtool_ctx *c, const char *fname) { fprintf(stderr, "%s: nmsg file output: %s\n", argv_program, fname); c->n_outputs += 1; + c->n_file_outputs += 1; } void @@ -692,24 +760,6 @@ add_pcapif_input(nmsgtool_ctx *c, nmsg_msgmod_t mod, const char *arg) { free(tmp); } -void -add_pres_input(nmsgtool_ctx *c, nmsg_msgmod_t mod, const char *fname) { - nmsg_input_t input; - nmsg_res res; - - input = nmsg_input_open_pres(open_rfile(fname), mod); - res = nmsg_io_add_input(c->io, input, NULL); - if (res != nmsg_res_success) { - fprintf(stderr, "%s: nmsg_io_add_input() failed\n", - argv_program); - exit(1); - } - if (c->debug >= 2) - fprintf(stderr, "%s: nmsg pres input: %s\n", argv_program, - fname); - c->n_inputs += 1; -} - void add_pres_output(nmsgtool_ctx *c, const char *fname) { nmsg_output_t output; @@ -744,7 +794,6 @@ add_pres_output(nmsgtool_ctx *c, const char *fname) { c->n_outputs += 1; } -#ifdef HAVE_JSON_C void add_json_input(nmsgtool_ctx *c, const char *fname) { nmsg_input_t input; @@ -762,15 +811,6 @@ add_json_input(nmsgtool_ctx *c, const char *fname) { fname); c->n_inputs += 1; } -#else /* HAVE_JSON_C */ -void -add_json_input(__attribute__((unused)) nmsgtool_ctx *c, - __attribute__((unused)) const char *fname) { - fprintf(stderr, "%s: Error: compiled without json-c support\n", - argv_program); - exit(EXIT_FAILURE); -} -#endif /* HAVE_JSON_C */ void add_json_output(nmsgtool_ctx *c, const char *fname) { diff --git a/src/nmsgtool.c b/src/nmsgtool.c index 638c14d82..0beefd29e 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -33,7 +33,14 @@ /* Globals. */ -static nmsgtool_ctx ctx; +/* + * Cull defaults. 0 is meaningful for both, so the options are read as strings + * and these stand until process_args() sees one given. + */ +static nmsgtool_ctx ctx = { + .zmin = NMSGTOOL_ZMIN_DEFAULT, + .zcull = NMSGTOOL_ZCULL_DEFAULT, +}; static argv_t args[] = { { 'b', "bpf", @@ -78,12 +85,6 @@ static argv_t args[] = { "endline", "continuation separator" }, - { 'f', "readpres", - ARGV_CHAR_P | ARGV_FLAG_ARRAY, - &ctx.r_pres, - "file", - "read pres format data from file" }, - { 'F', "filter", ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.filters, @@ -125,11 +126,7 @@ static argv_t args[] = { ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.r_json, "file", -#ifdef HAVE_JSON_C "read json format data from file" }, -#else /* HAVE_JSON_C */ - "read json format data from file (no support)" }, -#endif /* HAVE_JSON_C */ { 'J', "writejson", ARGV_CHAR_P | ARGV_FLAG_ARRAY, @@ -146,11 +143,11 @@ static argv_t args[] = { ARGV_CHAR_P, &ctx.kafka_key_field, "fieldname", -#if defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) +#if defined(HAVE_LIBRDKAFKA) "nmsg field for Kafka producer key" }, -#else /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#else /* defined(HAVE_LIBRDKAFKA) */ "nmsg field for Kafka producer key (no support)" }, -#endif /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#endif /* defined(HAVE_LIBRDKAFKA) */ {'\0', "readkafka", @@ -158,11 +155,7 @@ static argv_t args[] = { &ctx.r_kafka, "kafka", #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C - "read nmsg data from Kafka (binary or json)" }, -#else /* HAVE_JSON_C */ - "read nmsg containers from Kafka topic" }, -#endif /* HAVE_JSON_C */ + "read nmsg data from Kafka (nmsg, nmsgp, or json)" }, #else /* HAVE_LIBRDKAFKA */ "read nmsg data from Kafka topic (no support)" }, #endif /* HAVE_LIBRDKAFKA */ @@ -319,11 +312,7 @@ static argv_t args[] = { &ctx.w_kafka, "kafka", #ifdef HAVE_LIBRDKAFKA -#ifdef HAVE_JSON_C "write nmsg data to Kafka (binary or json)" }, -#else /* HAVE_JSON_C */ - "write nmsg containers to to Kafka topic" }, -#endif /* HAVE_JSON_C */ #else /* HAVE_LIBRDKAFKA */ "write nmsg data to Kafka topic (no support)" }, #endif /* HAVE_LIBRDKAFKA */ @@ -344,6 +333,24 @@ static argv_t args[] = { NULL, "compress nmsg output" }, + { '\0', "zasync", + ARGV_INT, + &ctx.zasync, + "n", + "compress file output on n threads (-1 auto)" }, + + { '\0', "zcull", + ARGV_CHAR_P, + &ctx.zcull_str, + "secs", + "drop a compressor idle after n secs (0 never)" }, + + { '\0', "zmin", + ARGV_CHAR_P, + &ctx.zmin_str, + "n", + "never drop below n compressors" }, + { ARGV_LAST, 0, 0, 0, 0, 0 } }; @@ -378,6 +385,13 @@ int main(int argc, char **argv) { ctx.statsmods_loaded = statsmod_vec_init(1); + /* Without it, outputs made before --zasync resolves get no pool. */ + ctx.initial_outputs = output_vec_init(1); + if (ctx.initial_outputs == NULL) { + fprintf(stderr, "%s: out of memory\n", argv_program); + exit(EXIT_FAILURE); + } + /* initialize the nmsg_io engine */ ctx.io = nmsg_io_init(); if (ctx.io == NULL) { @@ -449,14 +463,132 @@ usage(const char *msg) { exit(msg == NULL ? EXIT_SUCCESS : EXIT_FAILURE); } +/* + * How many compressor threads an output should get. A negative --zasync means + * auto: demand is about two workers per input, since a reader can saturate + * roughly one core compressing, bounded by the cores the readers leave. + */ +static unsigned +zworkers_count(const nmsgtool_ctx *c) { + long ncpu; + int n, spare; + + if (c->zasync == 0 || c->n_file_outputs == 0) + return (0); + if (c->zasync > 0) + return ((unsigned) c->zasync); + + ncpu = my_ncpu(); + + n = 2 * c->n_inputs; + + /* + * Split across the file outputs, which each get their own pool and + * share these cores. Not under --mirror, where every output is handed + * the whole stream and so needs the whole budget. + */ + if (!c->mirror) + n /= c->n_file_outputs; + + /* What the readers leave. An upper bound, but see the floor below. */ + spare = (int) ncpu - c->n_inputs; + if (spare < 1) + spare = 1; + if (n > spare) + n = spare; + + /* + * Applied last, and per output, so it overrides the bound above: a + * single input can carry several cores' worth on its own, which is + * what this floor is measured against. Oversubscribing costs nothing + * on an output that never saturates, since workers start on demand. + */ + if (n < NMSGTOOL_ZWORKERS_MIN) + n = NMSGTOOL_ZWORKERS_MIN; + + /* Never past what an explicit --zasync would be allowed. */ + if (n > (int) ncpu) + n = (int) ncpu; + + return ((unsigned) n); +} + +/* + * Cull policy first: setting the ceiling builds the pool, which reads it. The + * pool is an optimisation, so a failure to start one is reported, not fatal. + */ +static void +apply_zlib_workers(nmsgtool_ctx *c, nmsg_output_t output) { + nmsg_res res; + + nmsg_output_set_zlib_cull(output, c->zmin, c->zcull); + + res = nmsg_output_set_zlib_workers(output, c->zworkers_resolved); + if (res != nmsg_res_success) + fprintf(stderr, "%s: no compressor pool: %s\n", argv_program, + nmsg_res_lookup(res)); +} + +/* + * Resolve --zasync and apply it to the outputs that already exist. Deferred to + * the end of process_args() because the input count is not final until then: a + * channel alias (-C) expands to its sockets after the outputs are created. + */ +void +setup_nmsg_output_workers(nmsgtool_ctx *c) { + size_t i; + + c->zworkers_resolved = zworkers_count(c); + + if (c->initial_outputs != NULL) { + for (i = 0; i < output_vec_size(c->initial_outputs); i++) { + nmsg_output_t output = output_vec_data(c->initial_outputs)[i]; + + apply_zlib_workers(c, output); + } + output_vec_destroy(&c->initial_outputs); + } + + if (c->zworkers_resolved == 0) { + /* Nothing to compress on; say so rather than not. */ + if (c->zasync != 0) + fprintf(stderr, "%s: --zasync needs an nmsg file " + "output; ignored\n", argv_program); + else if (c->zcull != NMSGTOOL_ZCULL_DEFAULT || + c->zmin != NMSGTOOL_ZMIN_DEFAULT) + fprintf(stderr, "%s: --zcull and --zmin need --zasync " + "and an nmsg file output; ignored\n", argv_program); + return; + } + + if (c->debug >= 2) { + char cull[64]; + + if (c->zcull > 0) + snprintf(cull, sizeof(cull), + "culling to %d after %d idle second(s)", + c->zmin, c->zcull); + else + snprintf(cull, sizeof(cull), "never culling"); + + fprintf(stderr, "%s: compressing on up to %u thread(s) per " + "output, %s\n", argv_program, c->zworkers_resolved, cull); + } +} + void setup_nmsg_output(nmsgtool_ctx *c, nmsg_output_t output) { nmsg_output_set_buffered(output, !(c->unbuffered)); nmsg_output_set_endline(output, c->endline_str); nmsg_output_set_zlibout(output, c->zlibout); + apply_zlib_workers(c, output); nmsg_output_set_source(output, c->set_source); nmsg_output_set_operator(output, c->set_operator); nmsg_output_set_group(output, c->set_group); + + /* Outputs made before --zasync is resolved; see setup_nmsg_output_workers(). */ + if (c->initial_outputs != NULL) + output_vec_add(c->initial_outputs, output); } void diff --git a/src/nmsgtool.h b/src/nmsgtool.h index ef2ec41cf..263498796 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -39,6 +39,7 @@ #endif /* HAVE_LIBRDKAFKA */ #include "libmy/argv.h" +#include "libmy/my_cpu.h" #include "libmy/vector.h" union nmsgtool_sockaddr { @@ -49,14 +50,41 @@ union nmsgtool_sockaddr { typedef union nmsgtool_sockaddr nmsgtool_sockaddr; VECTOR_GENERATE(statsmod_vec, nmsg_statsmod_t) +VECTOR_GENERATE(output_vec, nmsg_output_t) + +/* + * Floor for an automatically chosen pool. One input socket can carry well over + * a core's worth of compression: at 120 MB/s it needed four workers, and two + * still lost 13 %. + */ +#define NMSGTOOL_ZWORKERS_MIN 4 + +/* + * Ceiling for an explicit --zasync. Compression is CPU-bound, so a thread per + * core is as far as it can help; workers start on demand, so this only bounds + * how far a saturated output may grow. libnmsg enforces the same ceiling, and + * sizes its reorder buffer a margin above it. + */ +#define NMSGTOOL_ZWORKERS_MAX(ncpu) (ncpu) + +/* + * Cull policy defaults. NMSGTOOL_ZMIN_DEFAULT is a floor on the live thread + * count, unrelated to NMSGTOOL_ZWORKERS_MIN above, which floors the ceiling. + */ +#define NMSGTOOL_ZMIN_DEFAULT 1 +#define NMSGTOOL_ZCULL_DEFAULT 300 typedef struct { /* parameters */ argv_array_t filters, statsmods; - argv_array_t r_nmsg, r_pres, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; + argv_array_t r_nmsg, r_kafka, r_sock, r_zsock, r_channel, r_zchannel, r_json; argv_array_t r_pcapfile, r_pcapif; argv_array_t w_nmsg, w_pres, w_sock, w_kafka, w_zsock, w_json; bool help, mirror, unbuffered, zlibout, daemon, version, interval_randomized; + char *zmin_str, *zcull_str; + int zasync; /* Compressor threads; -1 chooses. */ + int zmin; /* Compressors culling leaves. */ + int zcull; /* Idle seconds before a cull. */ char *endline, *kicker, *mname, *vname, *bpfstr, *filter_policy, *kafka_key_field; int debug, signal; unsigned mtu, count, interval, rate, freq, byte_rate; @@ -68,6 +96,9 @@ typedef struct { /* state */ char *endline_str; int n_inputs, n_outputs; + int n_file_outputs; /* Outputs a compressor pool can serve. */ + unsigned zworkers_resolved; /* 0 until process_args() finishes. */ + output_vec *initial_outputs; /* Non-NULL only during process_args(). */ nmsg_io_t io; #ifdef HAVE_LIBZMQ void *zmq_ctx; @@ -125,7 +156,6 @@ void add_file_input(nmsgtool_ctx *, const char *); void add_file_output(nmsgtool_ctx *, const char *); void add_pcapfile_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pcapif_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); -void add_pres_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pres_output(nmsgtool_ctx *, const char *); void add_json_input(nmsgtool_ctx *, const char *); void add_json_output(nmsgtool_ctx *, const char *); @@ -141,6 +171,7 @@ void pidfile_write(FILE *); void process_args(nmsgtool_ctx *); void setup_nmsg_input(nmsgtool_ctx *, nmsg_input_t); void setup_nmsg_output(nmsgtool_ctx *, nmsg_output_t); +void setup_nmsg_output_workers(nmsgtool_ctx *); void usage(const char *); #endif /* NMSGTOOL_H */ diff --git a/src/process_args.c b/src/process_args.c index 472adfe6c..8c73a45d7 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -63,6 +64,22 @@ droproot(nmsgtool_ctx *c, FILE *fp_pidfile) { argv_program, c->username); } +/* An integer in [min, max], or exit with 'what'. Rejects what atoi() would not. */ +static int +read_int_range(const char *str, int min, int max, const char *what) +{ + char *t; + long val; + + errno = 0; + val = strtol(str, &t, 0); + if (*str == '\0' || *t != '\0' || errno == ERANGE || + val < min || val > max) + usage(what); + + return (int) val; +} + /* Convert string to non-zero unsigned 32 bit val, returning zero on failure. */ static uint32_t read_uint32_nz(const char *str) @@ -125,6 +142,30 @@ process_args(nmsgtool_ctx *c) { if (c->mtu == 0) c->mtu = NMSG_WBUFSZ_JUMBO; + if (c->zasync < -1 || c->zasync > NMSGTOOL_ZWORKERS_MAX(my_ncpu())) + usage("--zasync must be -1 (choose), 0 (off), " + "or a thread count no greater than the available cores"); + + /* A container per message; a pool would spend a thread on each. */ + if (c->zasync != 0 && c->unbuffered) + usage("--zasync cannot be used with --unbuffered"); + + if (c->zcull_str != NULL) + c->zcull = read_int_range(c->zcull_str, 0, INT_MAX, + "--zcull must be 0 (never cull) or a number of seconds"); + + if (c->zmin_str != NULL) + c->zmin = read_int_range(c->zmin_str, 0, INT_MAX, + "--zmin must be 0 or a number of compressor threads"); + + /* + * Only checked against an explicit ceiling. Under --zasync -1 the + * ceiling is not settled until setup_nmsg_output_workers(), and libnmsg + * lowers the floor again to what the pool can run, reporting it at -dd. + */ + if (c->zasync > 0 && c->zmin > c->zasync) + usage("--zmin cannot exceed --zasync"); + if (c->vname == NULL && c->mname != NULL) c->vname = "base"; @@ -175,14 +216,14 @@ process_args(nmsgtool_ctx *c) { c->kicker = strdup(t); } -#if defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) +#if defined(HAVE_LIBRDKAFKA) /* kafka key */ if (c->kafka_key_field == NULL) { t = getenv("NMSG_KAFKA_KEY"); if (t != NULL) c->kafka_key_field = t; } -#endif /* defined(HAVE_LIBRDKAFKA) && defined(HAVE_JSON_C) */ +#endif /* defined(HAVE_LIBRDKAFKA) */ /* set source, operator, group */ if (c->set_source_str != NULL) { @@ -263,13 +304,11 @@ process_args(nmsgtool_ctx *c) { } /* -V, -T sanity check */ - if (ARGV_ARRAY_COUNT(c->r_pres) > 0 || - ARGV_ARRAY_COUNT(c->r_pcapfile) > 0 || + if (ARGV_ARRAY_COUNT(c->r_pcapfile) > 0 || ARGV_ARRAY_COUNT(c->r_pcapif) > 0) { if (c->vname == NULL || c->mname == NULL) - usage("reading presentation or pcap data requires " - "-V, -T"); + usage("reading pcap data requires -V, -T"); mod = nmsg_msgmod_lookup(c->vid, c->msgtype); if (mod == NULL) usage("unknown msgmod"); @@ -368,8 +407,7 @@ process_args(nmsgtool_ctx *c) { nmsg_chalias_free(&alias); } - /* pres inputs and outputs */ - process_args_loop_mod(c->r_pres, add_pres_input, mod); + /* pres outputs */ process_args_loop(c->w_pres, add_pres_output); /* json inputs and outputs */ @@ -412,6 +450,13 @@ process_args(nmsgtool_ctx *c) { add_pres_output(c, "-"); } + /* + * Size the pool now that every input exists. Must follow the implicit + * output above and precede daemonize(): pool threads start on first + * write, in whichever process does the writing. + */ + setup_nmsg_output_workers(c); + /* daemonize if necessary */ if (c->daemon) { if (!daemonize()) { diff --git a/tests/nmsg-dns-tests/test.sh.in b/tests/nmsg-dns-tests/test.sh.in index 24f76f77e..766aaa1b4 100755 --- a/tests/nmsg-dns-tests/test.sh.in +++ b/tests/nmsg-dns-tests/test.sh.in @@ -76,8 +76,6 @@ check read json base:dns and create json output cmp -s @abs_top_srcdir@/tests/nmsg-dns-tests/test2-dns.json @abs_top_builddir@/tests/nmsg-dns-tests/test2-dns.json.json.out check json-to-json -# NOTE: --readpres is not fully implemented for base:dns so aborts - # JSON input mistakes should result in no output $NMSGTOOL -dd -j @abs_top_srcdir@/tests/nmsg-dns-tests/test3-dns.json --writepres @abs_top_builddir@/tests/nmsg-dns-tests/test3-dns.json.pres.out 2>@abs_top_builddir@/tests/nmsg-dns-tests/test3-dns.json.pres.stderr.out check read broken json base:dns and create empty output diff --git a/tests/nmsg-dnsobs-tests/test.sh.in b/tests/nmsg-dnsobs-tests/test.sh.in index 0ae93a113..04a0186b5 100755 --- a/tests/nmsg-dnsobs-tests/test.sh.in +++ b/tests/nmsg-dnsobs-tests/test.sh.in @@ -54,6 +54,4 @@ check read json base:dnsobs and create base:dnsobs json output cmp -s ${SOURCE}.json ${OUTPUT}.json.json.out check json-to-json -# NOTE: --readpres is not fully implemented for base:dnsobs - exit $status diff --git a/tests/nmsg-dnsqr-tests/test.sh.in b/tests/nmsg-dnsqr-tests/test.sh.in index 4cbeb3c1f..20f83311e 100755 --- a/tests/nmsg-dnsqr-tests/test.sh.in +++ b/tests/nmsg-dnsqr-tests/test.sh.in @@ -84,6 +84,4 @@ check read nmsg base:dnsqr and generate pcap output using example cmp -s ${SOURCE}.pcap ${OUTPUT}.nmsg.pcap.out check example-nmsg-to-pcap -# NOTE: --readpres is not fully implemented for base:dnsqr so aborts - exit $status diff --git a/tests/nmsg-dnstap-tests/test.sh.in b/tests/nmsg-dnstap-tests/test.sh.in index 08093574a..eea0adebd 100755 --- a/tests/nmsg-dnstap-tests/test.sh.in +++ b/tests/nmsg-dnstap-tests/test.sh.in @@ -57,6 +57,5 @@ cmp -s ${SOURCE}.json ${OUTPUT}.nmsg.json.out check nmsg-to-json # NOTE: --readjson for base:dnstap is incomplete -# NOTE: --readpres is not fully implemented for base:dnstap exit $status diff --git a/tests/nmsg-http-tests/test.sh.in b/tests/nmsg-http-tests/test.sh.in index 812dba7e5..0084e719b 100755 --- a/tests/nmsg-http-tests/test.sh.in +++ b/tests/nmsg-http-tests/test.sh.in @@ -136,6 +136,4 @@ check read json base:http and create base:http json output cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.json.json.out check json-to-json -# NOTE: --readpres is not implemented for base:http - exit $status diff --git a/tests/test-private.c b/tests/test-private.c index 55a3bd382..ed75641f9 100644 --- a/tests/test-private.c +++ b/tests/test-private.c @@ -32,7 +32,7 @@ typedef int (*config_test)(struct config_file *); -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) typedef struct { const char *field; size_t length; @@ -178,7 +178,35 @@ test_kafka_key(void) { l_return_test_status(); } -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ + +/* Test null/invalid argument handling for nmsg_output_open_kafka_payload nmsg_input_open_kafka_payload. */ +static int +test_kafka_payload_papi(void) +{ + nmsg_output_t o; + + /* NULL address must return NULL without crashing. */ + o = nmsg_output_open_kafka_payload(NULL, NULL); + check(o == NULL); + + /* Address with no '@' is structurally invalid and must return NULL. */ + o = nmsg_output_open_kafka_payload("topic-no-broker", NULL); + check(o == NULL); + + nmsg_input_t i; + + /* NULL address must return NULL without crashing. */ + i = nmsg_input_open_kafka_payload(NULL); + check(i == NULL); + + /* Address with no '@' is structurally invalid and must return NULL. */ + i = nmsg_input_open_kafka_payload("topic-no-broker"); + check(i == NULL); + + l_return_test_status(); +} + +#endif /* HAVE_LIBRDKAFKA */ static int _test_config_file_papi_null(void) { @@ -337,10 +365,11 @@ main(void) check_explicit2_display_only(test_config_file() == 0, "test-private / test_config_file"); -#if (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) +#if (defined HAVE_LIBRDKAFKA) check_explicit2_display_only(test_kafka_papi() == 0, "test-private / test_kafka_papi"); check_explicit2_display_only(test_kafka_key() == 0, "test-private / test_kafka_key"); -#endif /* (defined HAVE_LIBRDKAFKA) && (defined HAVE_JSON_C) */ + check_explicit2_display_only(test_kafka_payload_papi() == 0, "test-private / test_kafka_payload_papi"); +#endif /* HAVE_LIBRDKAFKA */ g_check_test_status(false); } diff --git a/tests/test-zpool-cull.c b/tests/test-zpool-cull.c new file mode 100644 index 000000000..f2f8ffb9a --- /dev/null +++ b/tests/test-zpool-cull.c @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Compressors have to give their place back once they go idle, and the pool has + * to grow again afterwards. Linked against libnmsg's own objects, since the + * live worker count is not something the public headers expose. + * + * Only the first spawn is deterministic: a producer that finds no idle worker + * starts one, so a single write guarantees exactly one. Growing past that needs + * production to outrun compression, so nothing here asserts on it. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" +#include "private.h" + +#define BUFSZ NMSG_WBUFSZ_JUMBO +#define CULL_SECS 1 + +/* Long enough that a CULL_SECS deadline has certainly passed. */ +#define SETTLE_SECS 3 + +/* Payloads per burst. One container per burst is what the counts below rely on. */ +#define BURST 100 + +#if BURST * 64 > BUFSZ +#error "BURST no longer fits one container" +#endif + +/* Polling for a cull that has to happen, rather than guessing how long it takes. */ +#define POLL_STEP_MS 50 +#define POLL_MAX_MS 30000 + +static nmsg_msgmod_t mod; + +/* + * automake has no per-test timeout, so a wedged pool would hang forever. Set + * well above the runtime: under valgrind or a loaded builder this test is + * slower by an order of magnitude, and a watchdog that fires then is + * indistinguishable from the deadlock it is meant to catch. + */ +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-cull: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* Nothing useful to do; we are on our way out regardless. */ + } + _exit(1); +} + +/* nanosleep(), not sleep(): mixing sleep() with alarm() is unspecified. */ +static void +nap_ms(unsigned ms) +{ + struct timespec ts; + + ts.tv_sec = ms / 1000; + ts.tv_nsec = (long) (ms % 1000) * 1000000; + + while (nanosleep(&ts, &ts) != 0 && errno == EINTR) + ; +} + +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[1]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-cull: %s\n", what); + exit(1); +} + +static void +fail_count(const char *what, unsigned want, unsigned got) +{ + fprintf(stderr, "test-zpool-cull: %s: wanted %u, got %u\n", what, want, got); + exit(1); +} + +static nmsg_message_t +make_message(unsigned i) +{ + char payload[48]; + nmsg_message_t msg; + int len; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + len = snprintf(payload, sizeof(payload), "payload %u", i); + if (len < 0 || (size_t) len >= sizeof(payload)) + fail("snprintf() failed"); + + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + return (msg); +} + +static unsigned +count_payloads(const char *path) +{ + nmsg_input_t input; + nmsg_message_t msg; + unsigned n = 0; + int fd; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + for (;;) { + nmsg_res res = nmsg_input_read(input, &msg); + + if (res == nmsg_res_eof) + break; + if (res != nmsg_res_success) + fail("nmsg_input_read() failed"); + + nmsg_message_destroy(&msg); + n += 1; + } + + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ + + return (n); +} + +static nmsg_output_t +open_output(const char *path, unsigned workers, unsigned zmin, unsigned zcull) +{ + nmsg_output_t output; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + nmsg_output_set_zlib_cull(output, zmin, zcull); + nmsg_output_set_zlib_workers(output, workers); + + return (output); +} + +/* Write 'n' payloads and seal what they filled, so the pool sees a container. */ +static void +write_burst(nmsg_output_t output, unsigned n, unsigned tag) +{ + unsigned i; + + for (i = 0; i < n; i++) { + nmsg_message_t msg = make_message(tag + i); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + } + + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); +} + +static void +counts(nmsg_output_t output, unsigned *live, unsigned *peak, uint64_t *culled) +{ + struct nmsg_ostr_async *pool = output->stream->so_pool; + + if (pool == NULL) + fail("output has no compressor pool"); + + _output_async_counts(pool, live, peak, culled); +} + +/* Wait for the pool to reach 'want' live compressors, or give up and say so. */ +static void +wait_for_live(nmsg_output_t output, unsigned want) +{ + unsigned live, waited; + + for (waited = 0; waited < POLL_MAX_MS; waited += POLL_STEP_MS) { + counts(output, &live, NULL, NULL); + if (live == want) + return; + nap_ms(POLL_STEP_MS); + } + + counts(output, &live, NULL, NULL); + if (live != want) + fail_count("pool did not settle", want, live); +} + +/* + * A worker starts, goes quiet and gives its place back; the pool then grows + * again from empty. Reaching zero is what exercises the committer reaping a + * culled worker before its record can be spawned into again. + */ +static void +test_cull_to_empty(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, CULL_SECS); + unsigned live; + uint64_t culled; + + /* Polled, not read once: the worker arms its deadline as it finishes. */ + write_burst(output, BURST, 0); + wait_for_live(output, 1); + + wait_for_live(output, 0); + + counts(output, &live, NULL, &culled); + if (culled != 1) + fail_count("culls recorded", 1, (unsigned) culled); + + /* + * Two bursts: a culled worker's record is not free again until the + * committer reaps it, which happens on the next commit. Where the + * ceiling is 1 -- a single-core builder -- that commit is the first + * burst's, so the pool cannot grow until the second. + */ + write_burst(output, BURST, BURST); + write_burst(output, BURST, 2 * BURST); + wait_for_live(output, 1); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + if (count_payloads(path) != 3 * BURST) + fail_count("payloads written", 3 * BURST, count_payloads(path)); +} + +/* The floor is left alone, however long the pool stays quiet. */ +static void +test_floor(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 1, CULL_SECS); + unsigned live; + uint64_t culled; + + write_burst(output, BURST, 0); + nap_ms(SETTLE_SECS * 1000); + + counts(output, &live, NULL, &culled); + if (live != 1) + fail_count("floor not held", 1, live); + if (culled != 0) + fail_count("culls below the floor", 0, (unsigned) culled); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* Culling off leaves the pool at its high water mark. */ +static void +test_cull_disabled(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, 0); + unsigned live, peak; + uint64_t culled; + + write_burst(output, BURST, 0); + counts(output, &live, &peak, &culled); + if (live != peak) + fail_count("workers lost before settling", peak, live); + + nap_ms(SETTLE_SECS * 1000); + + counts(output, &live, &peak, &culled); + if (live != peak) + fail_count("culled with culling disabled", peak, live); + if (culled != 0) + fail_count("culls with culling disabled", 0, (unsigned) culled); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* + * A floor above the ceiling leaves nothing to cull, and the pool has to keep + * working rather than wedge or cull anyway. + * + * Not a test of async_min_workers_for()'s clamp, which cannot be one: nlive + * never exceeds the ceiling, so `nlive > min_workers` is false whether the + * floor was lowered or left at 8. The clamp's only effect is its -dd line. + */ +static void +test_floor_above_ceiling(const char *path) +{ + nmsg_output_t output = open_output(path, 1, 8, CULL_SECS); + unsigned live; + uint64_t culled; + + write_burst(output, BURST, 0); + nap_ms(SETTLE_SECS * 1000); + + counts(output, &live, NULL, &culled); + if (live != 1) + fail_count("clamped floor not held", 1, live); + if (culled != 0) + fail_count("culls under a clamped floor", 0, (unsigned) culled); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +/* + * Work keeps flowing across culls: containers written while the pool is + * shrinking and regrowing all have to arrive, in order. + */ +static void +test_traffic_across_culls(const char *path) +{ + nmsg_output_t output = open_output(path, 4, 0, CULL_SECS); + unsigned round, live; + + for (round = 0; round < 3; round++) { + write_burst(output, BURST, round * BURST); + wait_for_live(output, 0); + } + + counts(output, &live, NULL, NULL); + if (live != 0) + fail_count("pool did not settle", 0, live); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + if (count_payloads(path) != 3 * BURST) + fail_count("payloads written", 3 * BURST, count_payloads(path)); +} + +int +main(void) +{ + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char path[] = "/tmp/nmsg-zpool-cull.XXXXXX"; + int fd; + + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + /* mkstemp() only to get a unique name; the writers reopen by path. */ + fd = mkstemp(path); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + tmp_paths[0] = path; + atexit(unlink_tmp); + + test_cull_to_empty(path); + test_floor(path); + test_cull_disabled(path); + test_floor_above_ceiling(path); + test_traffic_across_culls(path); + + return (0); +} diff --git a/tests/test-zpool-mt.c b/tests/test-zpool-mt.c new file mode 100644 index 000000000..8fcc45d4b --- /dev/null +++ b/tests/test-zpool-mt.c @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * With several threads writing one output, the pool guarantees + * containers reach the file in the order they were + * sealed. A thread's own payloads must therefore appear in the file in the + * order that thread wrote them. + * + * This is a per-thread claim, not a global one. Which thread wins a given + * container is a race, so nothing is asserted about how threads interleave -- + * only that no thread's payloads are reordered among themselves. + * + * Without a pool, container_submit() writes on the calling thread and two + * threads race for w_lock in send_buffer(), so containers can reach the file + * out of order. The check below therefore applies to the pooled runs only. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" +#include "private.h" + +#define NUM_THREADS 4 +#define PER_THREAD 5000 +#define BUFSZ NMSG_WBUFSZ_JUMBO + +static nmsg_msgmod_t mod; +static nmsg_output_t output; + +struct writer { + pthread_t thr; + unsigned id; +}; + +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-mt: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* On our way out regardless. */ + } + _exit(1); +} + +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[1]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-mt: %s\n", what); + exit(1); +} + +static void * +writer_thread(void *arg) +{ + struct writer *w = (struct writer *) arg; + unsigned i; + + for (i = 0; i < PER_THREAD; i++) { + char payload[48]; + nmsg_message_t msg; + int len; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + len = snprintf(payload, sizeof(payload), "%u:%u", w->id, i); + if (len < 0 || (size_t) len >= sizeof(payload)) + fail("snprintf() failed"); + + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, + len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + } + + return (NULL); +} + +/* + * Read the file back and check that each thread's counters only ever increase. + * Returns the total number of payloads seen. + */ +static unsigned +verify_order(const char *path) +{ + unsigned last[NUM_THREADS]; + unsigned total = 0, i; + nmsg_input_t input; + nmsg_message_t msg; + int fd; + + for (i = 0; i < NUM_THREADS; i++) + last[i] = 0; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + while (nmsg_input_read(input, &msg) == nmsg_res_success) { + unsigned tid, seq; + void *data; + size_t len; + char buf[64]; + + if (nmsg_message_get_field(msg, "payload", 0, &data, + &len) != nmsg_res_success) + fail("nmsg_message_get_field() failed"); + + if (len >= sizeof(buf)) + fail("payload larger than expected"); + memcpy(buf, data, len); + buf[len] = '\0'; + + if (sscanf(buf, "%u:%u", &tid, &seq) != 2) + fail("payload did not parse"); + if (tid >= NUM_THREADS) + fail("payload carried an unknown thread id"); + + /* + * Counters start at 0, so 'last' holds the next value expected + * rather than the previous one seen. + */ + if (seq != last[tid]) { + fprintf(stderr, "test-zpool-mt: thread %u payload %u " + "arrived where %u was expected\n", + tid, seq, last[tid]); + exit(1); + } + last[tid] = seq + 1; + + nmsg_message_destroy(&msg); + total += 1; + } + + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ + + return (total); +} + +static void +run(const char *path, unsigned workers) +{ + struct writer writers[NUM_THREADS]; + unsigned i, total, peak = 0; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + if (nmsg_output_set_zlib_workers(output, workers) != nmsg_res_success) + fail("nmsg_output_set_zlib_workers() failed"); + + for (i = 0; i < NUM_THREADS; i++) { + writers[i].id = i; + if (pthread_create(&writers[i].thr, NULL, writer_thread, + &writers[i]) != 0) + fail("pthread_create() failed"); + } + + for (i = 0; i < NUM_THREADS; i++) + if (pthread_join(writers[i].thr, NULL) != 0) + fail("pthread_join() failed"); + + /* + * Read before the close, which takes the pool down. Without it the + * ordering check below would still pass with no pool at all -- the + * reordering a pool prevents is a race that need not occur. + */ + if (output->stream->so_pool == NULL) + fail("output has no compressor pool"); + _output_async_counts(output->stream->so_pool, NULL, &peak, NULL); + if (peak == 0) + fail("no compressor thread ever ran"); + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); + + total = verify_order(path); + if (total != NUM_THREADS * PER_THREAD) { + fprintf(stderr, "test-zpool-mt: read %u payloads, expected %u\n", + total, NUM_THREADS * PER_THREAD); + exit(1); + } +} + +int +main(void) +{ + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char path[] = "/tmp/nmsg-zpool-mt.XXXXXX"; + int fd; + + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + fd = mkstemp(path); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + tmp_paths[0] = path; + atexit(unlink_tmp); + + run(path, 1); + run(path, 4); + + return (0); +} \ No newline at end of file diff --git a/tests/test-zpool-order.c b/tests/test-zpool-order.c new file mode 100644 index 000000000..249e03831 --- /dev/null +++ b/tests/test-zpool-order.c @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2026 DomainTools LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The compressor pool must not change what gets written, only who compresses + * it. One producer writes the same payloads with a range of worker ceilings; + * every resulting file has to be byte for byte identical. + * + * A small bufsz is what makes this worth running: it puts many more containers + * in the file than a 1 MiB one would, so the reorder buffer wraps repeatedly + * and producers exercise the path where every worker is busy. The worker counts + * below span both buffer sizes, since the depth is derived from them. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nmsg.h" +#include "private.h" + +#define NUM_PAYLOADS 4000 +#define BUFSZ NMSG_WBUFSZ_JUMBO + +/* Flush and check the file part-way through, at this payload. */ +#define CHECKPOINT_AT 1500 + +/* Enable the pool at this payload in the late-enable run. */ +#define LATE_ENABLE_AT 700 + +static nmsg_msgmod_t mod; + +/* + * A wedged pool would otherwise hang the test suite forever: automake has no + * per-test timeout, so the test has to impose its own. + */ +static void +on_alarm(int sig __attribute__((unused))) +{ + static const char msg[] = "test-zpool-order: timed out\n"; + + if (write(STDERR_FILENO, msg, sizeof(msg) - 1) != sizeof(msg) - 1) { + /* Nothing useful to do; we are on our way out regardless. */ + } + _exit(1); +} + +/* Unlinked however the test ends, so a failure does not litter the tmp dir. */ +static const char *tmp_paths[2]; + +static void +unlink_tmp(void) +{ + unsigned i; + + for (i = 0; i < sizeof(tmp_paths) / sizeof(tmp_paths[0]); i++) { + if (tmp_paths[i] != NULL) + unlink(tmp_paths[i]); + } +} + +static void +fail(const char *what) +{ + fprintf(stderr, "test-zpool-order: %s\n", what); + exit(1); +} + +/* + * Payloads are deliberately short. A payload larger than bufsz would take the + * fragmenting path, whose fragment id is drawn from the RNG, and the output + * would then differ between two identical runs for reasons that have nothing + * to do with the pool. + */ +static nmsg_message_t +make_message(unsigned i) +{ + char payload[48]; + nmsg_message_t msg; + struct timespec ts; + size_t len; + int written; + + msg = nmsg_message_init(mod); + if (msg == NULL) + fail("nmsg_message_init() failed"); + + written = snprintf(payload, sizeof(payload), "payload %u", i); + if (written < 0 || (size_t) written >= sizeof(payload)) + fail("snprintf() failed"); + len = (size_t) written; + + if (nmsg_message_set_field(msg, "payload", 0, + (const uint8_t *) payload, len) != nmsg_res_success) + fail("nmsg_message_set_field() failed"); + + /* Fixed, so two runs cannot differ on the timestamp. */ + ts.tv_sec = 1000000000 + i; + ts.tv_nsec = 0; + nmsg_message_set_time(msg, &ts); + + return (msg); +} + +/* Payloads readable from a path right now. */ +static unsigned +count_payloads(const char *path) +{ + nmsg_input_t input; + nmsg_message_t msg; + unsigned n = 0; + int fd; + + fd = open(path, O_RDONLY); + if (fd < 0) + fail("open() for reading failed"); + + input = nmsg_input_open_file(fd); + if (input == NULL) + fail("nmsg_input_open_file() failed"); + + for (;;) { + nmsg_res res = nmsg_input_read(input, &msg); + + if (res == nmsg_res_eof) + break; + if (res != nmsg_res_success) + fail("nmsg_input_read() failed"); + + nmsg_message_destroy(&msg); + n += 1; + } + + nmsg_input_close(&input); /* Closes fd; autoclose is the default. */ + + return (n); +} + +/* + * Write the corpus to 'path'. + * + * workers is the ceiling handed to nmsg_output_set_zlib_workers(). late_enable + * defers that call until the stream is already part-written, which is only safe + * because the pool seeds itself from the stream's ticket counter. rate throttles + * the writer so the committer falls behind and producers have to wait on a slot. + */ +static void +write_corpus(const char *path, unsigned workers, bool late_enable, nmsg_rate_t rate) +{ + nmsg_output_t output; + unsigned i; + int fd; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + fail("open() for writing failed"); + + output = nmsg_output_open_file(fd, BUFSZ); + if (output == NULL) + fail("nmsg_output_open_file() failed"); + + nmsg_output_set_buffered(output, true); + nmsg_output_set_zlibout(output, true); + if (rate != NULL) + nmsg_output_set_rate(output, rate); + if (!late_enable) + nmsg_output_set_zlib_workers(output, workers); + + for (i = 0; i < NUM_PAYLOADS; i++) { + nmsg_message_t msg = make_message(i); + + if (nmsg_output_write(output, msg) != nmsg_res_success) + fail("nmsg_output_write() failed"); + nmsg_message_destroy(&msg); + + if (i == LATE_ENABLE_AT) { + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); + if (late_enable) + nmsg_output_set_zlib_workers(output, workers); + } + + /* + * A flush must leave everything written so far on disk, which + * is the contract every file rotation depends on. Checked while + * the output is still open, so it is the flush being tested and + * not the close. + */ + if (i == CHECKPOINT_AT) { + unsigned seen; + + if (nmsg_output_flush(output) != nmsg_res_success) + fail("nmsg_output_flush() failed"); + + seen = count_payloads(path); + if (seen != i + 1) { + fprintf(stderr, "test-zpool-order: flush left " + "%u of %u payloads on disk\n", seen, i + 1); + exit(1); + } + } + } + + /* + * Read before the close, which takes the pool down. Without this every + * assertion below would still hold if the pool had silently failed to + * start and everything had been compressed inline. + */ + if (workers > 0) { + unsigned peak = 0; + + if (output->stream->so_pool == NULL) + fail("output has no compressor pool"); + _output_async_counts(output->stream->so_pool, NULL, &peak, NULL); + if (peak == 0) + fail("no compressor thread ever ran"); + } + + if (nmsg_output_close(&output) != nmsg_res_success) + fail("nmsg_output_close() failed"); +} + +static void +compare(const char *ref, const char *path, const char *what) +{ + FILE *fa, *fb; + int ca, cb; + long off = 0; + + fa = fopen(ref, "rb"); + fb = fopen(path, "rb"); + if (fa == NULL || fb == NULL) + fail("fopen() for comparison failed"); + + do { + ca = getc(fa); + cb = getc(fb); + if (ca != cb) { + fprintf(stderr, "test-zpool-order: %s differs from the " + "inline output at byte %ld\n", what, off); + exit(1); + } + off += 1; + } while (ca != EOF); + + fclose(fa); + fclose(fb); +} + +int +main(void) +{ + /* + * 16 asks for a bigger reorder buffer than the rest; libnmsg clamps it + * to what the machine allows, which exercises the clamp either way. + */ + static const unsigned counts[] = { 1, 4, 8, 16 }; + /* static: unlink_tmp() runs from atexit(), after this frame is gone. */ + static char ref[] = "/tmp/nmsg-zpool-ref.XXXXXX"; + static char out[] = "/tmp/nmsg-zpool-out.XXXXXX"; + nmsg_rate_t rate; + unsigned i; + int fd; + + if (signal(SIGALRM, on_alarm) == SIG_ERR) + fail("signal() failed"); + alarm(600); + + if (nmsg_init() != nmsg_res_success) + fail("nmsg_init() failed"); + + mod = nmsg_msgmod_lookup_byname("base", "encode"); + if (mod == NULL) + fail("no base:encode message type"); + + /* mkstemp() only to get unique names; the writers reopen by path. */ + fd = mkstemp(ref); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + tmp_paths[0] = ref; + + fd = mkstemp(out); + if (fd < 0) + fail("mkstemp() failed"); + close(fd); + tmp_paths[1] = out; + + atexit(unlink_tmp); + + /* No pool: the reference every other run has to match. */ + write_corpus(ref, 0, false, NULL); + + if (count_payloads(ref) != NUM_PAYLOADS) + fail("reference output is missing payloads"); + + for (i = 0; i < sizeof(counts) / sizeof(counts[0]); i++) { + char what[64]; + + snprintf(what, sizeof(what), "output with %u worker(s)", counts[i]); + write_corpus(out, counts[i], false, NULL); + compare(ref, out, what); + } + + /* Pool enabled once the stream is already part-written. */ + write_corpus(out, 4, true, NULL); + compare(ref, out, "output with the pool enabled mid-stream"); + + /* + * Throttled, so the committer lags and the reorder buffer fills. This is + * the only run that reaches the slot wait in _output_async_submit(). + */ + rate = nmsg_rate_init(200, 1); + if (rate == NULL) + fail("nmsg_rate_init() failed"); + write_corpus(out, 4, false, rate); + compare(ref, out, "rate-limited output"); + nmsg_rate_destroy(&rate); + + return (0); +} \ No newline at end of file