From 8b42d6b28b6465da73c798577f15ac7b6105bdbc Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=A9mie=20Galarneau?= Date: Tue, 3 Jan 2023 18:41:23 -0500 Subject: [PATCH] Fix: sessiond: instance uuid is not sufficiently unique MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Observed issue ============== Tracing a cluster of machines -- all launched simultaneously -- to the same relay daemon occasionally produces corrupted traces. The size of packets received (as seen from the relay daemon logs) and that of those present in the on-disk stream occasionally didn't match. The traces were observed to all relate to the same trace UUID, but present packet begin timestamps that were not monotonic for a given stream. This causes both Babeltrace 1.x and 2.x to fail to open the traces with different error messages related to clocks. Cause ===== On start, the session daemon generates a UUID to uniquely identify the sessiond instance. Since the UUID generation utils use time() to seed the random number generator, two session daemons launched within the same second can end up with the same instance UUID. Since the relay daemon relies on this UUID to uniquely identify a session daemon accross its various connections, identifier clashes can cause streams from the same `uid` or `pid` to become scrambled resulting in corrupted traces. Solution ======== The UUID utils now initializes its random seed using the getrandom() API in non-blocking mode. If that fails -- most likely because the random pool is depleted or the syscall is not available on the platform -- it falls back to using a hash of two time readings (with nanosecond precision), of the hostname, and the PID. Known drawbacks =============== This fix implements many fallbacks, each with their own caveats and we don't have full test coverage for all of those for the moment. This article presents the different drawbacks of using /dev/urandom vs getrandom(). https://lwn.net/Articles/884875/ As for the pseudo-random time and configuration based fallback, it is meant as a last resort for platforms or configurations where both getrandom() (old kernels or non-Linux platforms) and /dev/urandom (e.g. locked-down container) are not be available. I haven't done a formal analysis of the entropy of this home-grown method. The practical use-case we want to enable is launching multiple virtual machines (or containers) at roughly the same time and ensure that they don't end up using the same sessiond UUID. In that respect, having a different host name and minute timing changes seem enough to prevent a UUID clash. Using the PID as part of the hash also helps when launching multiple session daemons simultaneously for different users. Change-Id: I320fff7bc52752ff504643569e49fa3c02472ec2 Signed-off-by: Jérémie Galarneau --- configure.ac | 4 +- src/common/Makefile.am | 1 + src/common/random.c | 181 +++++++++++++++++++++++++++++++++++++++++ src/common/random.h | 28 +++++++ src/common/uuid.c | 18 ++-- tests/unit/test_uuid.c | 5 ++ 6 files changed, 226 insertions(+), 11 deletions(-) create mode 100644 src/common/random.c create mode 100644 src/common/random.h diff --git a/configure.ac b/configure.ac index bf4c45350..c7e7e6cd8 100644 --- a/configure.ac +++ b/configure.ac @@ -223,7 +223,7 @@ AC_CHECK_HEADERS([ \ signal.h stdlib.h sys/un.h sys/socket.h stdlib.h stdio.h \ getopt.h sys/ipc.h sys/shm.h popt.h grp.h arpa/inet.h \ netdb.h netinet/in.h paths.h stddef.h sys/file.h sys/ioctl.h \ - sys/mount.h sys/param.h sys/time.h elf.h + sys/mount.h sys/param.h sys/time.h elf.h sys/random.h sys/syscall.h ]) AM_CONDITIONAL([HAVE_ELF_H], [test x$ac_cv_header_elf_h = xyes]) @@ -235,7 +235,7 @@ AC_CHECK_FUNCS([ \ mkdir munmap putenv realpath rmdir socket strchr strcspn strdup \ strncasecmp strndup strnlen strpbrk strrchr strstr strtol strtoul \ strtoull dirfd gethostbyname2 getipnodebyname epoll_create1 \ - sched_getcpu sysconf sync_file_range + sched_getcpu sysconf sync_file_range getrandom ]) # Check if clock_gettime, timer_create, timer_settime, and timer_delete are available in lib rt, and if so, diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 723e6d16c..b0c761a1a 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -47,6 +47,7 @@ libcommon_lgpl_la_SOURCES = \ mi-lttng.c mi-lttng.h \ notification.c \ notify.c \ + random.c random.h \ readwrite.c readwrite.h \ runas.c runas.h \ session-consumed-size.c \ diff --git a/src/common/random.c b/src/common/random.c new file mode 100644 index 000000000..6b5995c10 --- /dev/null +++ b/src/common/random.c @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2023 Jérémie Galarneau + * + * SPDX-License-Identifier: LGPL-2.1-only + * + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef HAVE_SYS_SYSCALL_H +#include +#endif + +/* getrandom is available in Linux >= 3.17. */ +#if defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) + +#include + +/* A glibc wrapper is provided only for glibc >= 2.25. */ +#if defined(HAVE_GETRANDOM) +/* Simply use the existing wrapper, passing the non-block flag. */ +static ssize_t _call_getrandom_nonblock(char *out_data, size_t size) +{ + return getrandom(out_data, size, GRND_NONBLOCK); +} +#else +static ssize_t _call_getrandom_nonblock(char *out_data, size_t size) +{ + const int grnd_nonblock_flag = 0x1; + long ret = syscall(SYS_getrandom, out_data, size, grnd_nonblock_flag); + + if (ret < 0) { + errno = -ret; + ret = -1; + } + + return ret; +} +#endif /* defined(HAVE_GETRANDOM) */ + +/* Returns either with a full read or throws. */ +static int getrandom_nonblock(char *out_data, size_t size) +{ + /* + * Since GRND_RANDOM is _not_ used, a partial read can only be caused + * by a signal interruption. In this case, retry. + */ + int ret = 0; + ssize_t random_ret; + + do { + random_ret = _call_getrandom_nonblock(out_data, size); + } while ((random_ret > 0 && random_ret != size) || (random_ret == -1 && errno == EINTR)); + + if (random_ret < 0) { + PERROR("Failed to get true random data using getrandom(): size=%zu", size); + ret = -1; + } + + return ret; +} +#else /* defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) */ +static int getrandom_nonblock(char *out_data, size_t size) +{ + WARN("getrandom() is not supported by this platform"); + return -1; +} +#endif /* defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) */ + +static int produce_pseudo_random_seed(seed_t *out_seed) +{ + int ret; + struct timespec real_time = {}; + struct timespec monotonic_time = {}; + unsigned long hash_seed; + char hostname[LTTNG_HOST_NAME_MAX] = {}; + unsigned long seed; + unsigned long pid; + + ret = clock_gettime(CLOCK_REALTIME, &real_time); + if (ret) { + PERROR("Failed to read real time while generating pseudo-random seed"); + goto error; + } + + ret = clock_gettime(CLOCK_MONOTONIC, &monotonic_time); + if (ret) { + PERROR("Failed to read monotonic time while generating pseudo-random seed"); + goto error; + } + + ret = gethostname(hostname, sizeof(hostname)); + if (ret) { + PERROR("Failed to get host name while generating pseudo-random seed"); + goto error; + } + + hash_seed = (unsigned long) real_time.tv_nsec ^ (unsigned long) real_time.tv_sec ^ + (unsigned long) monotonic_time.tv_nsec ^ + (unsigned long) monotonic_time.tv_sec; + seed = hash_key_ulong((void *) real_time.tv_sec, hash_seed); + seed ^= hash_key_ulong((void *) real_time.tv_nsec, hash_seed); + seed ^= hash_key_ulong((void *) monotonic_time.tv_sec, hash_seed); + seed ^= hash_key_ulong((void *) monotonic_time.tv_nsec, hash_seed); + + pid = getpid(); + seed ^= hash_key_ulong((void *) pid, hash_seed); + seed ^= hash_key_str(hostname, hash_seed); + ret = 0; + + *out_seed = (seed_t) seed; +error: + return ret; +} + +static int produce_random_seed_from_urandom(seed_t *out_seed) +{ + int ret = 0, read_ret; + const int urandom_raw_fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + + if (urandom_raw_fd < 0) { + PERROR("Failed to open `/dev/urandom`"); + ret = -1; + goto end; + } + + read_ret = lttng_read(urandom_raw_fd, out_seed, sizeof(*out_seed)); + if (read_ret != sizeof(*out_seed)) { + PERROR("Failed to read from `/dev/urandom`: size=%zu", + sizeof(*out_seed)); + ret = -1; + goto end; + } + +end: + if (urandom_raw_fd >= 0) { + if (close(urandom_raw_fd)) { + PERROR("Failed to close `/dev/urandom` file descriptor"); + } + } + return ret; +} + +int lttng_produce_true_random_seed(seed_t *out_seed) +{ + return getrandom_nonblock((char *) out_seed, sizeof(*out_seed)); +} + +int lttng_produce_best_effort_random_seed(seed_t *out_seed) +{ + int ret; + + ret = lttng_produce_true_random_seed(out_seed); + if (!ret) { + goto end; + } else { + WARN("Failed to produce a random seed using getrandom(), falling back to pseudo-random device seed generation which will block until its pool is initialized"); + } + + ret = produce_random_seed_from_urandom(out_seed); + if (!ret) { + goto end; + } else { + WARN("Failed to produce a random seed from the urandom device"); + } + + /* Fallback to seed generation based on time and system configuration. */ + ret = produce_pseudo_random_seed(out_seed); +end: + return ret; +} diff --git a/src/common/random.h b/src/common/random.h new file mode 100644 index 000000000..eb5bcfa0c --- /dev/null +++ b/src/common/random.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 Jérémie Galarneau + * + * SPDX-License-Identifier: LGPL-2.1-only + * + */ + +#ifndef LTTNG_RANDOM_H +#define LTTNG_RANDOM_H + +#include + +typedef unsigned int seed_t; + +/* + * Get a seed from a reliable source of randomness without blocking. Returns 0 + * on success, -1 on failure. + */ +int lttng_produce_true_random_seed(seed_t *out_seed); + +/* + * Get a random seed making a best-effort to use a true randomness source, + * but falling back to a pseudo-random seed based on the time and various system + * configuration values on failure. Returns 0 on success, -1 on failure. + */ +int lttng_produce_best_effort_random_seed(seed_t *out_seed); + +#endif /* LTTNG_RANDOM_H */ diff --git a/src/common/uuid.c b/src/common/uuid.c index 26fb61e99..0e01c2185 100644 --- a/src/common/uuid.c +++ b/src/common/uuid.c @@ -7,6 +7,9 @@ */ #include +#include +#include + #include #include #include @@ -78,18 +81,15 @@ int lttng_uuid_generate(lttng_uuid uuid_out) } if (!lttng_uuid_is_init) { - /* - * We don't need cryptographic quality randomness to - * generate UUIDs, seed rand with the epoch. - */ - const time_t epoch = time(NULL); - - if (epoch == (time_t) -1) { - ret = -1; + seed_t new_seed; + + ret = lttng_produce_best_effort_random_seed(&new_seed); + if (ret) { + ERR("Failed to initialize random seed while generating UUID"); goto end; } - srand(epoch); + srand(new_seed); lttng_uuid_is_init = true; } diff --git a/tests/unit/test_uuid.c b/tests/unit/test_uuid.c index e13a09751..8193746c1 100644 --- a/tests/unit/test_uuid.c +++ b/tests/unit/test_uuid.c @@ -41,6 +41,11 @@ static const char invalid_str_4[] = "2d-6c6d756574-470e-9142-a4e6ad03f143"; static const char invalid_str_5[] = "4542ad19-9e4f-4931-8261-2101c3e089ae7"; static const char invalid_str_6[] = "XX0123"; +/* For error.h */ +int lttng_opt_quiet = 1; +int lttng_opt_verbose = 0; +int lttng_opt_mi; + static void run_test_lttng_uuid_from_str(void) { -- 2.34.1