When I execute /etc/init.d/haveged stop then haveged throws segfault. # gdb -q /usr/sbin/haveged 'core-1588341382-0-11-!usr!sbin!haveged-4584' Reading symbols from /usr/sbin/haveged... Reading symbols from /usr/lib64/debug//usr/sbin/haveged.debug... [New LWP 4584] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/sbin/haveged -r 0 -w 1024 -v 1 -p /run/haveged.pid'. Program terminated with signal SIGSEGV, Segmentation fault. #0 __GI___libc_free (mem=0x7f630c066000) at malloc.c:3109 3109 p = mem2chunk (mem); (gdb) bt #0 __GI___libc_free (mem=0x7f630c066000) at malloc.c:3109 #1 0x00007f630c035f9e in havege_destroy (hptr=0x564b56b6b900) at havege.c:197 #2 0x0000564b561c3dca in error_exit (format=<optimized out>) at haveged.c:708 #3 0x0000564b561c33f1 in run_daemon (argv=0x7ffce7627758, path=0x564b56b6b2a0 "/usr/sbin/haveged", h=0x564b56b6b900) at haveged.c:573 #4 main (argc=<optimized out>, argv=0x7ffce7627758) at haveged.c:470 (gdb) bt full [13/1877] #0 __GI___libc_free (mem=0x7f630c066000) at malloc.c:3109 ar_ptr = <optimized out> p = <optimized out> hook = 0x0 #1 0x00007f630c035f9e in havege_destroy (hptr=0x564b56b6b900) at havege.c:197 htemp = <optimized out> temp = <optimized out> #2 0x0000564b561c3dca in error_exit (format=<optimized out>) at haveged.c:708 buffer = "Stopping due to signal 15\n\000s/system/cpu/cpu4/cache", '\000' <repeats 1246 times>... ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0x7ffce76261c0, reg_save_area = 0x7ffce7626100}} #3 0x0000564b561c33f1 in run_daemon (argv=0x7ffce7627758, path=0x564b56b6b2a0 "/usr/sbin/haveged", h=0x564b56b6b900) at haveged.c:573 current = 2 write_fd = {fds_bits = {32, 0 <repeats 15 times>}} read_fd = {fds_bits = {8, 0 <repeats 15 times>}} nbytes = <optimized out> r = <optimized out> max = <optimized out> random_fd = 5 conn_fd = <optimized out> output = 0x7f630c066000 stat_buf = {st_dev = 6, st_ino = 1031, st_nlink = 1, st_mode = 8630, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 264, st_size = 0, st_blksize = 4096, st_blocks = 0, st_atim = { tv_sec = 1588235982, tv_nsec = 860000000}, st_mtim = {tv_sec = 1588235982, tv_nsec = 860000000}, st_ctim = {tv_sec = 1588235982, tv_nsec = 860000000}, __glibc_reserved = {0, 0, 0}} random_fd = <optimized out> conn_fd = <optimized out> output = <optimized out> stat_buf = <optimized out> current = <optimized out> nbytes = <optimized out> r = <optimized out> max = <optimized out> write_fd = <optimized out> read_fd = <optimized out> __d0 = <optimized out> __d1 = <optimized out> __d0 = <optimized out> __d1 = <optimized out> __d = <optimized out> __d = <optimized out> __d = <optimized out> two = <optimized out> rc = <optimized out> __d = <optimized out> __d = <optimized out> __d = <optimized out> #4 main (argc=<optimized out>, argv=0x7ffce7627758) at haveged.c:470 path = 0x564b56b6b2a0 "/usr/sbin/haveged" arg0 = <optimized out> cmds = {0x564b561c50c4 "b", 0x564b561c5243 "buffer", 0x564b561c524a "1", 0x564b561c5588 "Buffer size [KW], default: 128", 0x564b561c522b "d", 0x564b561c524c "data", 0x564b561c524a "1", 0x564b561c55a8 "Data cache size [KB], with fallback to: 16", 0x564b561c5251 "c", 0x564b561c5253 "command", 0x564b561c524a "1", 0x564b561c55d8 "Send a command mode to an already running haveged", 0x564b561c51db "i", 0x564b561c525b "inst", 0x564b561c524a "1", 0x564b561c5610 "Instruction cache size [KB], with fallback to: 16", 0x564b561c5260 "f", 0x564b561c52a5 "file", 0x564b561c524a "1", 0x564b561c5648 "Sample output file, default: 'sample', '-' for stdout", 0x564b561c5262 "F", 0x564b561c5264 "Foreground", 0x564b561c526f "0", 0x564b561c5271 "Run daemon in foreground", 0x564b561c5293 "r", 0x564b561c528a "run", 0x564b561c524a "1", 0x564b561c5680 "0=daemon, 1=config info, >1=<r>KB sample", 0x564b561c5144 "n", 0x564b561c528e "number", 0x564b561c524a "1", 0x564b561c56b0 "Output size in [k|m|g|t] bytes, 0 = unlimited to stdout", 0x564b561c5295 "o", 0x564b561c5297 "onlinetest", 0x564b561c524a "1", 0x564b561c56e8 "[t<x>][c<x>] x=[a[n][w]][b[w]] 't'ot, 'c'ontinuous, default: ta8b", 0x564b561c52dd "p", 0x564b561c52a2 "pidfile", 0x564b561c524a "1", 0x564b561c5730 "daemon pidfile, default: /var/run/haveged.pid", 0x564b561c50bf "s", 0x564b561c52aa "source", 0x564b561c524a "1", 0x564b561c5760 "Injection source file, default: 'data', '-' for stdin", 0x564b561c52a0 "t", 0x564b561c52bb "threads", 0x564b561c524a "1", 0x564b561c52b1 "Number of threads", 0x564b561c52c3 "v", 0x564b561c52c5 "verbose", 0x564b561c524a "1", 0x564b561c5798 "Verbose mask 0=none,1=summary,2=retries,4=timing,8=loop,16=code,32=test", 0x564b561c51d8 "w", 0x564b561c52cd "write", 0x564b561c524a "1", 0x564b561c57e0 "Set write_wakeup_threshold [bits]", 0x564b561c52d3 "h", 0x564b561c52da "help", 0x564b561c526f "0", 0x564b561c52d5 "This help"} long_options = {{name = 0x564b561c5243 "buffer", has_arg = 1, flag = 0x0, val = 98}, {name = 0x564b561c524c "data", has_arg = 1, flag = 0x0, val = 100}, {name = 0x564b561c5253 "command", has_arg = 1, flag = 0x0, val = 99}, {name = 0x564b561c525b "inst", has_arg = 1, flag = 0x0, val = 105}, {name = 0x564b561c52a5 "file", has_arg = 1, flag = 0x0, val = 102}, { name = 0x564b561c5264 "Foreground", has_arg = 0, flag = 0x0, val = 70}, {name = 0x564b561c528a "run", has_arg = 1, flag = 0x0, val = 114}, {name = 0x564b561c528e "number", has_arg = 1, flag = 0x0, val = 110}, {name = 0x564b561c5297 "onlinetest", has_arg = 1, flag = 0x0, val = 111}, {name = 0x564b561c52a2 "pidfile", has_arg = 1, flag = 0x0, val = 112}, { name = 0x564b561c52bb "threads", has_arg = 1, flag = 0x0, val = 116}, {name = 0x564b561c52c5 "verbose", has_arg = 1, flag = 0x0, val = 118}, {name = 0x564b561c52cd "write", has_arg = 1, flag = 0x0, val = 119}, {name = 0x564b561c52da "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}, {name = 0x0, has_arg = -412977552, flag = 0x564b561c3790 <_start>, val = 1444684948}} short_options = "b:d:c:i:f:Fr:n:o:p:t:v:w:h\000\000\000\000" --Type <RET> for more, q to quit, c to continue without paging-- c = <optimized out> i = <optimized out> j = <optimized out> bufct = <optimized out> bufrem = <optimized out> ierr = <optimized out> cmd = {ioSz = 8200, collectSize = 0, icacheSize = 0, dcacheSize = 0, options = 1, nCores = 0, metering = 0x0, msg_out = 0x564b561c3890 <print_msg>, injection = 0x0, procFs = 0x0, sysFs = 0x0, testSpec = 0x564b561c50c6 "ta8bcb"} (gdb) Reproducible: Always # emerge --info haveged Portage 2.3.89 (python 3.6.10-final-0, default/linux/amd64/17.0/no-multilib/hardened, gcc-9.3.0, glibc-2.29-r7, 5.3.0-89600-gd7dbd2ce12c4 x86_64) ================================================================= System Settings ================================================================= System uname: Linux-5.3.0-89600-gd7dbd2ce12c4-x86_64-Intel-R-_Xeon-R-_CPU_E3-1230_v5_@_3.40GHz-with-gentoo-2.6 KiB Mem: 7618228 total, 567112 free KiB Swap: 1548284 total, 1548284 free Timestamp of repository gentoo: Fri, 01 May 2020 10:43:54 +0000 Head commit of repository gentoo: 52ce6865c875509367f6544136cc4ab699eafd86 sh bash 4.4_p23-r1 ld GNU ld (Gentoo 2.33.1 p2) 2.33.1 ccache version 3.7.7 [disabled] app-shells/bash: 4.4_p23-r1::gentoo dev-lang/perl: 5.30.1::gentoo dev-lang/python: 2.7.17-r2::gentoo, 3.6.10-r1::gentoo, 3.7.7-r1::gentoo dev-util/ccache: 3.7.7-r1::gentoo dev-util/cmake: 3.16.5::gentoo dev-util/pkgconfig: 0.29.2::gentoo sys-apps/baselayout: 2.6-r1::gentoo sys-apps/openrc: 0.42.1::gentoo sys-apps/sandbox: 2.13::gentoo sys-devel/autoconf: 2.69-r4::gentoo sys-devel/automake: 1.15.1-r2::gentoo, 1.16.1-r1::gentoo sys-devel/binutils: 2.33.1-r1::gentoo sys-devel/gcc: 9.3.0::gentoo sys-devel/gcc-config: 2.2.1::gentoo sys-devel/libtool: 2.4.6-r6::gentoo sys-devel/make: 4.2.1-r4::gentoo sys-kernel/linux-headers: 5.4::gentoo (virtual/os-headers) sys-libs/glibc: 2.29-r7::gentoo Repositories: gentoo location: /usr/portage sync-type: git sync-uri: https://github.com/gentoo-mirror/gentoo.git sync-user: portage:portage priority: -1000 Installed sets: @masscheck, @nagios-plugins, @sheepdog ACCEPT_KEYWORDS="amd64" ACCEPT_LICENSE="*" CBUILD="x86_64-pc-linux-gnu" CFLAGS="-O3 -pipe -march=westmere -mtune=native -fno-unwind-tables -fuse-linker-plugin -ftracer -fuse-ld=gold -fvar-tracking-assignments" CHOST="x86_64-pc-linux-gnu" CONFIG_PROTECT="/etc /usr/share/easy-rsa /usr/share/gnupg/qualified.txt /var/bind" CONFIG_PROTECT_MASK="/etc/ca-certificates.conf /etc/env.d /etc/fonts/fonts.conf /etc/gconf /etc/gentoo-release /etc/php/apache2-php7.2/ext-active/ /etc/php/apache2-php7.3/ext-active/ /etc/php/apache2-php7.4/ ext-active/ /etc/php/cgi-php7.2/ext-active/ /etc/php/cgi-php7.3/ext-active/ /etc/php/cgi-php7.4/ext-active/ /etc/php/cli-php7.2/ext-active/ /etc/php/cli-php7.3/ext-active/ /etc/php/cli-php7.4/ext-active/ /et c/revdep-rebuild /etc/sandbox.d /etc/terminfo" CXXFLAGS="-O3 -pipe -march=westmere -mtune=native -fno-unwind-tables -fuse-linker-plugin -ftracer -fuse-ld=gold -fvar-tracking-assignments" DISTDIR="/usr/portage/distfiles" ENV_UNSET="DBUS_SESSION_BUS_ADDRESS DISPLAY GOBIN PERL5LIB PERL5OPT PERLPREFIX PERL_CORE PERL_MB_OPT PERL_MM_OPT XAUTHORITY XDG_CACHE_HOME XDG_CONFIG_HOME XDG_DATA_HOME XDG_RUNTIME_DIR" FCFLAGS="-O3 -pipe -march=westmere -mtune=native -fno-unwind-tables -fuse-linker-plugin -ftracer -fuse-ld=gold -fvar-tracking-assignments" FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs cgroup collision-protect compressdebug config-protect-if-modified distlocks ebuild-locks fixlafiles ipc-sandbox multilib-strict network-s andbox news parallel-fetch parallel-install pid-sandbox preserve-libs protect-owned qa-unresolved-soname-deps sandbox sfperms splitdebug strict unmerge-logs unmerge-orphans userfetch userpriv usersandbox use rsync xattr" FFLAGS="-O3 -pipe -march=westmere -mtune=native -fno-unwind-tables -fuse-linker-plugin -ftracer -fuse-ld=gold -fvar-tracking-assignments" GENTOO_MIRRORS="http://distfiles.gentoo.org" LANG="pl_PL.utf8" LDFLAGS="-Wl,-O1 -Wl,--as-needed -Wl,--sort-common" LINGUAS="en" MAKEOPTS="-j4 -l5" PKGDIR="/var/cache/binpkgs" PORTAGE_CONFIGROOT="/" PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --omit-dir-times --compress --force --whole-file --delete --stats --human-readable --timeout=180 --exclude=/distfiles --exclude=/local --e xclude=/packages --exclude=/.git" PORTAGE_TMPDIR="/var/tmp" USE="acl aio amd64 apache2 bash-completion bzip2 caps crypt hardened iconv idn ipv6 jit libtirpc lto modules nano-syntax ncurses nls nptl openmp pcre pie readline seccomp smp split-usr ssl ssp threads unicod e vhosts vim-syntax xattr xtpax zlib" ABI_X86="64" ADA_TARGET="gnat_2018" APACHE2_MODULES="alias authn_core access_compat auth_basic authz_core authn_alias authn_anon auth_digest authn_dbm authn_default auth n_file authz_dbm authz_default authz_groupfile authz_host authz_owner authz_user autoindex cgi dav dav_fs dav_lock dir env expires ext_filter filter headers hugepages include info log_config logio mime mime_ magic negotiation remoteip rewrite setenvif status unique_id unixd socache_shmcb usertrack vhost_alias" APACHE2_MPMS="itk" CALLIGRA_FEATURES="karbon sheets words" COLLECTD_PLUGINS="apache aggregation bind cg roups conntrack contextswitch cpu curl curl_json df disk dns email entropy ethstat exec filecount hugepages interface iptables irq lvm match_regex mysql netlink load memory network nginx notify_email ntpd op envpn ping postgresql processes protocols redis rrdcached rrdtool snmp statsd swap syslog tail tail_csv tcpconns unixsock uptime vmem" CPU_FLAGS_X86="aes mmx mmxext sse sse2 sse3 ssse3" ELIBC="glibc" GRUB_PL ATFORMS="pc" KERNEL="linux" L10N="en" LIBREOFFICE_EXTENSIONS="presenter-console presenter-minimizer" NGINX_MODULES_HTTP="access autoindex browser charset fastcgi gzip map limit_zone proxy rewrite http2 stub_ status gzip_static" OFFICE_IMPLEMENTATION="libreoffice" PHP_TARGETS="php7-2" POSTGRES_TARGETS="postgres10 postgres11" PYTHON_SINGLE_TARGET="python3_6" PYTHON_TARGETS="python2_7 python3_6" RUBY_TARGETS="ruby2 4 ruby25" USERLAND="GNU" XTABLES_ADDONS="fuzzy geoip lscan psd Xsysrq tarpit" Unset: CC, CPPFLAGS, CTARGET, CXX, EMERGE_DEFAULT_OPTS, INSTALL_MASK, LC_ALL, PORTAGE_BINHOST, PORTAGE_BUNZIP2_COMMAND, PORTAGE_COMPRESS, PORTAGE_COMPRESS_FLAGS, PORTAGE_RSYNC_EXTRA_OPTS ================================================================= Package Settings ================================================================= sys-apps/haveged-1.9.8c::gentoo was built with the following: USE="threads (-selinux) -static-libs" CFLAGS="-O2 -march=westmere -mtune=native -ggdb3 -gdwarf-4 -fvar-tracking-assignments -ftracer -pipe" CXXFLAGS="-O2 -march=westmere -mtune=native -ggdb3 -gdwarf-4 -fvar-tracking-assignments -ftracer -pipe" FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs cgroup collision-protect compressdebug config-protect-if-modified distlocks ebuild-locks fixlafiles installsources ipc-sandbox multilib-s trict network-sandbox news parallel-fetch parallel-install pid-sandbox preserve-libs protect-owned qa-unresolved-soname-deps sandbox sfperms splitdebug strict unmerge-logs unmerge-orphans userfetch userpriv usersandbox usersync xattr"
I have the same issue...
My guess it's a memory corruption in haveged. Looking at the backtrace it's somehow related to signal handler itself. As it it was a double-free or similar. If you run the daemon under valgrind (or build it with ASAN), can you get more detailed log where corruption initially happens?
Reproduced locally with sys-apps/haveged[threads]. Should be able to debug.
# valgrind --track-origins=yes --trace-children=yes --num-callers=50 --malloc-fill=0xE1 --free-fill=0xF1 --log-file=/tmp/lf /usr/sbin/haveged -w 1024 -v 1 -p /run/haveged.pid -r 0 # kill -TERM 2318700 The relevant output is: ==2318675== For lists of detected and suppressed errors, rerun with: -s ==2318675== ERROR SUMMARY: 14 errors from 2 contexts (suppressed: 0 from 0) ==2318700== Invalid free() / delete / delete[] / realloc() ==2318700== at 0x48389CB: free (in /usr/lib64/valgrind/vgpreload_memcheck-amd64-linux.so) ==2318700== by 0x48AED9F: havege_destroy (havege.c:197) ==2318700== by 0x10BCA5: error_exit (haveged.c:708) ==2318700== by 0x10B1B7: run_daemon (haveged.c:573) ==2318700== by 0x10B1B7: main (haveged.c:470) ==2318700== Address 0x4845000 is in a rw- anonymous segment It's a mismatch in mmap/free().
(In reply to Sergei Trofimovich from comment #4) > ==2318675== For lists of detected and suppressed errors, rerun with: -s > ==2318675== ERROR SUMMARY: 14 errors from 2 contexts (suppressed: 0 from 0) > ==2318700== Invalid free() / delete / delete[] / realloc() > ==2318700== at 0x48389CB: free (in > /usr/lib64/valgrind/vgpreload_memcheck-amd64-linux.so) > ==2318700== by 0x48AED9F: havege_destroy (havege.c:197) > ==2318700== by 0x10BCA5: error_exit (haveged.c:708) > ==2318700== by 0x10B1B7: run_daemon (haveged.c:573) > ==2318700== by 0x10B1B7: main (haveged.c:470) > ==2318700== Address 0x4845000 is in a rw- anonymous segment > > It's a mismatch in mmap/free(). Allocation/free rules of 'h_ptr->io_buf' are a bit tricky: https://github.com/jirka-h/haveged/blob/master/src/havege.c havege_create() { #if NUMBER_CORES>1 havege_ipc(h, n, sz); #else h->io_buf = malloc(sz); ... } void havege_destroy( /* RETURN: none */ H_PTR hptr) /* IN-OUT: app anchor */ { if (NULL != hptr) { H_COLLECT *htemp; void *temp; #if NUMBER_CORES>1 if (!havege_exit(hptr)) return; /* only main thread continues */ #endif if (0 != (temp=hptr->io_buf)) { hptr->io_buf = 0; free(temp); } ... This 'free(temp);' should not be called for '#if NUMBER_CORES>1' Either it belong in an '#else' branch, or 'havege_exit(hptr)' should do a cleanup and looks like it tries to do the cleanup. It's not clear from the code how it can happen.
Created attachment 635554 [details, diff] haveged-1.9.8c-threaded-shutdown.patch I think I have a fix. daemonize() breaks the assumption that main process does not change the pid. That breaks the invariant of how main process should look like. Main process ended up thinking it's a shigle thread and used single-thread deallocation. Can you try a workaround attached? It seems to work for me.
Also proposed fix upstream as https://github.com/jirka-h/haveged/pull/30
Created attachment 635556 [details, diff] haveged-1.9.8c-threaded-shutdown.patch Updated patch to handle --disable-threads case.
Merged upstream.
