OpenBSD Mail Box: June 2020

Tuesday, June 30, 2020

Re: [NEW] sysutils/fluent-bit

Hi,

From: Landry Breuil <landry@openbsd.org>
Date: 2020-05-27 5:47:33

> On Wed, May 27, 2020 at 01:48:55PM +0900, Masato Asou wrote:
>> Hi,
>>
>> This is a new port sysutils/fluent-bit. Fluent bit is a Log Processor
>> and Forwarder.
>
> Great stuff, i think i had a look at it some years ago and it looked
> very-linuxy/not that much portable.. nice that you made it.
>
> some notes:
> * why USE_NINJA=No ?
> * why not taking maintainership ?
> * it feels weird to force gmake usage w/ cmake.. in which voodooo
> trickery does cmake end up generating $< -gmake rules ? cant the cmake
> stuff be fixed instead ?
> * 1.4.5 was released 2 days ago
> * many portability patches without comments.. are you planning to push
> those upstream via github ?
>
> other than that it reads good at first sight.
>
> Landry

The original version has been updated from 1.4.4 to 1.4.6.

The difference at the tail end is a comparison with the .tgz attached
to the email below.

ok?

From: Masato Asou <asou@soum.co.jp>
Date: Mon, 01 Jun 2020 17:07:34 +0900 (JST)

> From: Stuart Henderson <stu@spacehopper.org>
> Date: 2020-05-27 10:38:26
>
>> On 2020/05/27 07:47, Landry Breuil wrote:
>>> On Wed, May 27, 2020 at 01:48:55PM +0900, Masato Asou wrote:
>>> > Hi,
>>> >
>>> > This is a new port sysutils/fluent-bit. Fluent bit is a Log Processor
>>> > and Forwarder.
>>>
>>> Great stuff, i think i had a look at it some years ago and it looked
>>> very-linuxy/not that much portable.. nice that you made it.
>>>
>>> some notes:
>>> * why USE_NINJA=No ?
>>
>> Couple of reasons. One is that they don't want the full path in traces and are
>> using some gmake-specific mess to define __FILENAME__ with the 'base' source
>> directory stripped off (that's easily avoided by using __FILE__ instead).
>> Another is that they bundle various dependencies and are building them by
>> calling $(MAKE). Also some of those dependencies use gmake-specific things.
>
> </snip>
>
> I've applied your patch.
>
> How about this patch.

diff --git a/sysutils/fluent-bit/Makefile b/sysutils/fluent-bit/Makefile
index 622f4aaea80..d4f2dae8abb 100644
--- a/sysutils/fluent-bit/Makefile
+++ b/sysutils/fluent-bit/Makefile
@@ -2,13 +2,15 @@

COMMENT = fast log processor and forwarder

-GH_ACCOUNT = fluent
-GH_PROJECT = fluent-bit
-GH_TAGNAME = v1.4.4
+VERSION = 1.4
+V = ${VERSION}.6

CATEGORIES = sysutils

+DISTNAME = fluent-bit-${V}
+EXTRACT_SUFX = .tar.gz
HOMEPAGE = https://fluentbit.io/
+MASTER_SITES = ${HOMEPAGE:=releases/${VERSION}/}

#MAINTAINER =

diff --git a/sysutils/fluent-bit/distinfo b/sysutils/fluent-bit/distinfo
index e3e4e31ba17..8adc6e0442b 100644
--- a/sysutils/fluent-bit/distinfo
+++ b/sysutils/fluent-bit/distinfo
@@ -1,2 +1,2 @@
-SHA256 (fluent-bit-1.4.4.tar.gz) = f9abc86ca56fbfc7b5c3fff1511f21e6eaa4c75cae7690e2fcca381872abf6f7
-SIZE (fluent-bit-1.4.4.tar.gz) = 12614962
+SHA256 (fluent-bit-1.4.6.tar.gz) = 4da02a9d0f42a82eab07ea3e5713007c0fd5539485da133515ae464789fc11fa
+SIZE (fluent-bit-1.4.6.tar.gz) = 12509211
diff --git a/sysutils/fluent-bit/patches/patch-plugins_in_tail_tail_file_c b/sysutils/fluent-bit/patches/patch-plugins_in_tail_tail_file_c
index 98028f8e44c..408b2e887c1 100644
--- a/sysutils/fluent-bit/patches/patch-plugins_in_tail_tail_file_c
+++ b/sysutils/fluent-bit/patches/patch-plugins_in_tail_tail_file_c
@@ -3,7 +3,7 @@ $OpenBSD$
Index: plugins/in_tail/tail_file.c
--- plugins/in_tail/tail_file.c.orig
+++ plugins/in_tail/tail_file.c
-@@ -1041,9 +1041,8 @@
+@@ -1053,9 +1053,8 @@
{
int ret;
char *buf;
@@ -15,7 +15,7 @@ Index: plugins/in_tail/tail_file.c
#elif defined(__APPLE__)
char path[PATH_MAX];
#elif defined(_MSC_VER)
-@@ -1056,21 +1055,24 @@
+@@ -1068,21 +1067,24 @@
return NULL;
}

diff --git a/sysutils/fluent-bit/patches/patch-src_CMakeLists_txt b/sysutils/fluent-bit/patches/patch-src_CMakeLists_txt
index aef2a395fcc..a73f2dc2428 100644
--- a/sysutils/fluent-bit/patches/patch-src_CMakeLists_txt
+++ b/sysutils/fluent-bit/patches/patch-src_CMakeLists_txt
@@ -3,7 +3,7 @@ $OpenBSD$
Index: src/CMakeLists.txt
--- src/CMakeLists.txt.orig
+++ src/CMakeLists.txt
-@@ -191,6 +191,13 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+@@ -192,6 +192,13 @@
)
endif()

diff --git a/sysutils/fluent-bit/patches/patch-src_flb_scheduler_c b/sysutils/fluent-bit/patches/patch-src_flb_scheduler_c
index 539e9efafdb..19b75431bca 100644
--- a/sysutils/fluent-bit/patches/patch-src_flb_scheduler_c
+++ b/sysutils/fluent-bit/patches/patch-src_flb_scheduler_c
@@ -3,7 +3,7 @@ $OpenBSD$
Index: src/flb_scheduler.c
--- src/flb_scheduler.c.orig
+++ src/flb_scheduler.c
-@@ -376,7 +376,7 @@ int flb_sched_event_handler(struct flb_config *config,
+@@ -378,7 +378,7 @@
}
else if (timer->type == FLB_SCHED_TIMER_FRAME) {
sched = timer->data;
diff --git a/sysutils/fluent-bit/patches/patch-src_fluent_bit_c b/sysutils/fluent-bit/patches/patch-src_fluent_bit_c
index a8444c7d7f7..42b578b8017 100644
--- a/sysutils/fluent-bit/patches/patch-src_fluent_bit_c
+++ b/sysutils/fluent-bit/patches/patch-src_fluent_bit_c
@@ -3,7 +3,7 @@ $OpenBSD$
Index: src/fluent-bit.c
--- src/fluent-bit.c.orig
+++ src/fluent-bit.c
-@@ -379,14 +379,8 @@ static void flb_signal_handler(int signal)
+@@ -382,14 +382,8 @@
case SIGQUIT:
case SIGHUP:

Re: strlcpy version speed tests?

#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <err.h>
#include <sys/param.h>

/* cc strlcpy_test.c -pipe -O2 -o strlcpy_test && ./strlcpy_testfast */

/*
* Copy string src to buffer dst of size dsize. At most dsize-1
* chars will be copied. Always NUL terminates (unless dsize == 0).
* Returns strlen(src); if retval >= dsize, truncation occurred.
*/
static size_t
strlcpy0(char *dst, const char *src, size_t dsize)
{
const char *osrc = src;
size_t nleft = dsize;

/* Copy as many bytes as will fit. */
if (nleft != 0) {
while (--nleft != 0) {
if ((*dst++ = *src++) == '\0')
break;
}
}

/* Not enough room in dst, add NUL and traverse rest of src. */
if (nleft == 0) {
if (dsize != 0)
*dst = '\0'; /* NUL-terminate dst */
while (*src++)
;
}

return(src - osrc - 1); /* count does not include NUL */
}

static size_t
strlcpy3(char *dst, const char *src, size_t dsize)
{
const char *osrc = src;
size_t nleft = dsize;

if (nleft != 0) {
/* Copy as many bytes as will fit. */
while (--nleft != 0)
if ((*dst++ = *src++) == '\0')
return(src - osrc - 1);
*dst = '\0';
}

/* Not enough room in dst, traverse rest of src. */
while (*src++)
;

return(src - osrc - 1); /* count does not include NUL */
}

static size_t
strlcpy4(char dst[], const char src[], size_t dsize)
{
const char *osrc = src;
size_t nleft = dsize;

if (nleft != 0) {
if (--nleft == 0) {
*dst = '\0'; /* NUL-terminate dst */
if (*src == '\0')
return 0;
} else {
/* Copy as many bytes as will fit. */
if ((*dst = *src) == '\0')
return 0;
while (--nleft != 0)
if ((*++dst = *++src) == '\0')
return(src - osrc);
dst[1] = '\0'; /* NUL-terminate dst */
}
} else if (*src == '\0')
return 0;

/* Not enough room in dst, traverse rest of src. */
while (*++src)
;

return(src - osrc); /* count does not include NUL */
}

int main()
{

long double cpu_time_used;
size_t y;
struct timespec tv_start, tv_end;
char *buffer, *buffer2;

size_t n = 50000;
size_t m = n + 50;

buffer = malloc(m);
if (buffer == NULL) err(1, "malloc");
buffer2 = malloc(n);
if (buffer2 == NULL) err(1, "malloc");

/* no intermediate '\0' */
for (y = 0; y < m; ++y)
buffer[y] = arc4random_uniform(255) + 1;
buffer[m - 1] = '\0';

clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_start);
strlcpy(buffer2, buffer, n);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_end);

cpu_time_used =
(long double) (tv_end.tv_sec - tv_start.tv_sec) +
(long double) (tv_end.tv_nsec - tv_start.tv_nsec) /
(long double) 1000000000;

printf("\n\nstrlcpy\n");
printf("time = %.9Lf\n\n\n", cpu_time_used);

clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_start);
strlcpy0(buffer2, buffer, n);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_end);

cpu_time_used =
(long double) (tv_end.tv_sec - tv_start.tv_sec) +
(long double) (tv_end.tv_nsec - tv_start.tv_nsec) /
(long double) 1000000000;

printf("\n\nstrlcpy0\n");
printf("time = %.9Lf\n\n\n", cpu_time_used);

clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_start);
strlcpy3(buffer2, buffer, n);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_end);

cpu_time_used =
(long double) (tv_end.tv_sec - tv_start.tv_sec) +
(long double) (tv_end.tv_nsec - tv_start.tv_nsec) /
(long double) 1000000000;

printf("\n\nstrlcpy3\n");
printf("time = %.9Lf\n\n\n", cpu_time_used);

clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_start);
strlcpy4(buffer2, buffer, n);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &tv_end);

cpu_time_used =
(long double) (tv_end.tv_sec - tv_start.tv_sec) +
(long double) (tv_end.tv_nsec - tv_start.tv_nsec) /
(long double) 1000000000;

printf("\n\nstrlcpy4\n");
printf("time = %.9Lf\n\n\n", cpu_time_used);

return 0;
}
I suppose this strlcpy4 without a goto is more elegant
-Luke

On Tue, Jun 30, 2020 at 10:07 PM Luke Small <lukensmall@gmail.com> wrote:

> I made it SUPER easy to test my assertion. The code is there. No
> configuration needed.
>
> On Tue, Jun 30, 2020 at 9:59 PM Theo de Raadt <deraadt@openbsd.org> wrote:
>
>> Luke Small <lukensmall@gmail.com> wrote:
>>
>> > So did you run the program on one of those?
>>
>> Why would I?
>>
>> i see a sales pitch
>>
>> and i go BULLSHIT
>>
>> and I'm done
>>
>> --
> -Luke
>

Re: strlcpy version speed tests?

On 1/7/20 11:18 am, Luke Small wrote:
> I made a couple different versions if anybody is interested!

You don't need our permission…
--
Stuart Longland (aka Redhatter, VK4MSL)

I haven't lost my mind...
...it's backed up on a tape somewhere.

Re: net/flow-tools man pages, seg fault on improper command line input

> Thanks, I've just fixed /etc/flow-tools/flow-tools and
> /usr/usr/local/bin/python2.7 in -current.

Thank you very much for your quick assistance!

I did find a few new problems:

Category: net/flow-tools
Environment:
Details : OpenBSD 6.7-current (GENERIC.MP) #306: Fri Jun 26 22:13:55 MDT 2020
deraadt@amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP
Architecture: OpenBSD.amd64
Description:

1. These man pages show formatting codes instead of man pages andrunning make shows errors with docbook-to-man:
flow-capture
flow-export
flow-header
flow-import
flow-nfilter
flow-report
flow-stat
flow-tools-examples
flow-xlate

2. Improper command line input can generate a segmentation fault.

How-To-Repeat:
1. man flow-capture, flow-export, ...
To generate the make error: cd /usr/ports/net/flow-tools && make

snipped output:

Making all in docs
gmake[1]: Entering directory '/usr/ports/pobj/flow-tools-0.68.6/flow-tools-0.68.6/docs'
docbook-to-man flow-capture.sgml > flow-capture.1
nsgmls:/usr/local/share/sgml//docbook/dsssl/modular/catalog:28:0:W: DTDDECL catalog entries are not supported
Abort trap (core dumped)
docbook-to-man flow-cat.sgml > flow-cat.1
nsgmls:/usr/local/share/sgml//docbook/dsssl/modular/catalog:28:0:W: DTDDECL catalog entries are not supported

2. flow-cat ft-v05.xxx | flow-print -f -6
generates segmentation fault. The above extra hyphen for the 6 is a deliberate error.
flow-print -f 6 gives IP Accounting format.

Fix:
1. not sure. man pages are okay in previous versions

2. Instead of using atoi, perhaps use strtonum and errx in the getopt cases. There are
many calls to atoi, but I just tried to fix one in flow-print. I was unable to test
it though. I tried just removing doc from the Makefile MAKE_FLAGS SUBDIRS and apply
the patch below, but when I ran make install, it always seemed to install the code
without my patch. I'm sure that part is my fault, I just don't know how to correct it.
I tried:
cd /usr/ports/net/flow-tools
make patch
cd /usr/ports/pobj/flow-tools-0.68.6/flow-tools-0.68.6/src/flow-print.c
cp flow-print.c flow-print.c.orig
# edit file, then
cd /usr/ports/net/flow-tools
make update-patches
cd patches
cvs add patch-src-_flow-print_c
cd /usr/ports/net/flow-tools
cvs diff -uNp
make
make install

There may be other examples of input that would generate a segmentation fault.

My patch for what it's worth:

$OpenBSD$

Index: src/flow-print.c
--- src/flow-print.c.orig
+++ src/flow-print.c
@@ -28,6 +28,7 @@

#include "ftconfig.h"
#include <ftlib.h>
+#include <err.h>

#include <sys/time.h>
#include <sys/types.h>
@@ -98,6 +99,7 @@ char **argv;
int i, format_index, set_format, ret;
int print_header, options, debug;
char cc; /* comment character */
+ const char *errstr;

/* init fterr */
fterr_setid(argv[0]);
@@ -124,7 +126,9 @@ char **argv;
break;

case 'f': /* format */
- format_index = atoi(optarg);
+ format_index = strtonum(optarg, 0, NFORMATS, &errstr);
+ if (errstr != NULL)
+ errx(1,"invalid report format number. %s : %s", errstr, optarg);
set_format = 1;
break;

strlcpy version speed tests?

#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <err.h>
#include <sys/param.h>

/* cc strlcpy_test.c -pipe -O2 -o strlcpy_test && ./strlcpy_testfast */

/* Copy as many bytes as will fit. */
if (nleft != 0) {
while (--nleft != 0) {
if ((*dst++ = *src++) == '\0')
break;
}
}

/* Not enough room in dst, add NUL and traverse rest of src. */
if (nleft == 0) {
if (dsize != 0)
*dst = '\0'; /* NUL-terminate dst */
while (*src++)
;
}

return(src - osrc - 1); /* count does not include NUL */
}

static size_t
strlcpy3(char *dst, const char *src, size_t dsize)
{
const char *osrc = src;
size_t nleft = dsize;

return(src - osrc - 1); /* count does not include NUL */
}

static size_t
strlcpy4(char dst[], const char src[], size_t dsize)
{
const char *osrc = src;
size_t nleft = dsize;

if (nleft != 0) {
if (--nleft == 0)
{
*dst = '\0';
if (*src == '\0')
return 0;
goto strlcpy_jump;
}
/* Copy as many bytes as will fit. */
if ((*dst = *src) == '\0')
return 0;
while (--nleft != 0)
if ((*++dst = *++src) == '\0')
return(src - osrc);
dst[1] = '\0'; /* NUL-terminate dst */
} else if (*src == '\0')
return 0;

strlcpy_jump:

/* Not enough room in dst, traverse rest of src. */
while (*++src)
;

return(src - osrc); /* count does not include NUL */
}

int main()
{

long double cpu_time_used;
size_t y;
struct timespec tv_start, tv_end;
char *buffer, *buffer2;

size_t n = 50000;
size_t m = n + 500;

buffer = malloc(m);
if (buffer == NULL) err(1, "malloc");
buffer2 = malloc(n);
if (buffer2 == NULL) err(1, "malloc");

/* no intermediate '\0' */
for (y = 0; y < m; ++y)
buffer[y] = arc4random_uniform(255) + 1;
buffer[m - 1] = '\0';