From 3460c4f7fc9ad81c04809fea2129d1c7c10d0933 Mon Sep 17 00:00:00 2001 From: Kienan Stewart Date: Fri, 8 Nov 2024 15:59:43 -0500 Subject: [PATCH] lttng: Warn on session start when client-visible shm path is too small MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Observed issue ============== When adding one or more channels to a session, it is possible to configure sessions such that the minimum memory required to successfully trace exceeds the size of the shared memory location. There is no user feedback when starting sessions that are configured in such a way that they will never work. It therefore leads to user confusion when no events are recorded. Steps to reproduce ================== On a system with 4 cpus: ``` sudo mount -t tmpfs -o size=64M tmpfs /mnt lttng create --shm-path=/mnt lttng enable-channel -u --subbuf-size=8M a0 lttng enable-event -u --all -c a0 lttng start lttng stop ``` Cause ===== As allocations are made lazily, there is no feedback for this type of scenario visible to an lttng client. The session can successfully start; however, there will be allocation errors visible in the consumer daemon logs. Solution ======== When starting a session, the minimum shared memory size required is estimated using the configured non-kernel channels. This assumes per-PID or per-UID buffers with a single PID or a single UID. An extra set of buffers are estimated when the session is configured in snapshot mode. E.g. ``` $ lttng create Session auto-20241112-102937 created. Traces will be output to /home/kstewart/src/efficios/lttng/master/home//lttng-traces/auto-20241112-102937 $ lttng enable-channel -u --subbuf-size=256M --num-subbuf=8 a2 user space channel `a2` enabled for session `auto-20241112-102937` $ lttng start Warning: The estimated minimum shared memory size for all non-kernel channels of session 'auto-20241112-102937' is greater than the total shared memory allocated to the default shared memory location (8192MiB >= 7873MiB). Tracing for this session may not record events due to allocation failures. Tracing started for session `auto-20241112-102937` $ lttng list Available recording sessions: 1) auto-20241112-102937 [active] Trace output: /home/kstewart/src/efficios/lttng/master/home//lttng-traces/auto-20241112-102937 ``` Known drawbacks =============== The intention of this change is to be backwards compatible, and therefore doesn't stop the session from being started. With per-PID or per-UID sub-buffers, the estimate is based only a single PID or UID. Additional PIDs or UIDs could cause allocation failures later on in the runtime which will continue to only be visible in the consumer daemon logs. The shm path is estimated by the lttng client rather than the consumer daemon. It is possible for the consumer daemon be running in a different container whose shared memory path maps to a different backing storage. This checks only against the total size of the shared memory storage - not the current available usage. It does not take into account other current enabled sessions. Change-Id: I589ec461ec4a89b72bcdf1271065591ac471acca Signed-off-by: Signed-off-by: Kienan Stewart Signed-off-by: Jérémie Galarneau --- src/bin/lttng/commands/start.cpp | 170 ++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 4 deletions(-) diff --git a/src/bin/lttng/commands/start.cpp b/src/bin/lttng/commands/start.cpp index f00ce973e..c4d5d4ee3 100644 --- a/src/bin/lttng/commands/start.cpp +++ b/src/bin/lttng/commands/start.cpp @@ -11,16 +11,23 @@ #include "../utils.hpp" #include +#include #include +#include #include +#include +#include + +#include #include #include #include #include -#include +#include +#include +#include #include -#include enum { OPT_HELP = 1, @@ -75,6 +82,157 @@ end: return ret; } +namespace { +unsigned long estimate_session_minimum_shm_size(const struct lttng_session *session) +{ + unsigned long est_min_size{ 0 }; + int domain_count{ 0 }; + int ncpus{ 0 }; + + ncpus = utils_get_cpu_count(); /* Throws on error */ + auto domains = [&session, &domain_count]() { + struct lttng_domain *raw_domains = nullptr; + domain_count = lttng_list_domains(session->name, &raw_domains); + if (domain_count < 0) { + LTTNG_THROW_ERROR(lttng::format("Failed to list domains for session '%s'", + session->name)); + } + return lttng::make_unique_wrapper( + raw_domains); + }(); + + for (auto domain_x = 0; domain_x < domain_count; domain_x++) { + switch (domains.get()[domain_x].type) { + case LTTNG_DOMAIN_UST: + case LTTNG_DOMAIN_JUL: + case LTTNG_DOMAIN_LOG4J: + case LTTNG_DOMAIN_PYTHON: + case LTTNG_DOMAIN_LOG4J2: + break; + default: + DBG("Domain %d not supported for shm estimation", + domains.get()[domain_x].type); + continue; + } + auto handle = [&session, &domains, domain_x] { + struct lttng_handle *raw_handle = nullptr; + raw_handle = lttng_create_handle(session->name, &domains.get()[domain_x]); + if (!raw_handle) { + LTTNG_THROW_ERROR( + lttng::format("Failed to get lttng handle for session '%s'", + session->name)); + } + return lttng::make_unique_wrapper( + raw_handle); + }(); + + int channel_count{ 0 }; + auto channels = [&handle, &session, &channel_count] { + struct lttng_channel *raw_channels = nullptr; + channel_count = lttng_list_channels(handle.get(), &raw_channels); + if (channel_count < 0) { + LTTNG_THROW_ERROR(lttng::format( + "Failed to list channels for session '%s'", session->name)); + } + return lttng::make_unique_wrapper( + raw_channels); + }(); + + for (int channel_x = 0; channel_x < channel_count; channel_x++) { + auto channel = &channels.get()[channel_x]; + /* + * This assumes per-uid or per-pid buffers with a minimum of one uid + * or pid. + */ + est_min_size += ((ncpus + session->snapshot_mode) * + channel->attr.num_subbuf * channel->attr.subbuf_size); + } + } + + return est_min_size; +} + +void warn_on_small_client_shm(const char *session_name) +{ + constexpr const char *CLIENT_SHM_TEST_PATH = "/lttng-client-fake"; + const auto session_spec = + lttng::cli::session_spec(lttng::cli::session_spec::type::NAME, session_name); + const auto sessions = lttng::cli::list_sessions(session_spec); + + if (sessions.size() != 1) { + LTTNG_THROW_ERROR(lttng::format("No sessions found for name '{}'", session_name)); + } + + const char *shm_path{}; + unsigned long estimated_size{ 0 }; + unsigned long memfd_device_size{ 0 }; + auto ret = lttng_get_session_shm_path_override(&sessions[0], &shm_path); + if (ret == LTTNG_GET_SESSION_SHM_PATH_STATUS_INVALID_PARAMETER) { + LTTNG_THROW_ERROR( + lttng::format("Failed to get session '{}' shm path, return code: {}", + session_name, + int(ret))); + } + + struct statvfs statbuf; + if (ret == LTTNG_GET_SESSION_SHM_PATH_STATUS_OK) { + /* Have to turn the shm_path into parent, since it doesn't yet exist. */ + const auto path = std::string{ shm_path }; + const auto parent = path.substr(0, path.find_last_of('/')); + DBG("Session '%s' shm_path is set to '%s', using parent '%s'", + session_name, + shm_path, + parent.c_str()); + if (statvfs(parent.c_str(), &statbuf) != 0) { + LTTNG_THROW_POSIX( + lttng::format( + "Failed to get the capacity of the filesystem at path '{}'", + parent.c_str()), + errno); + } + } else { + /* + * The shm_path is whatever gets used by the OS when opening + * an anonymous shm fd. POSIX doesn't provide an introspection + * into it. + */ + auto fd = [CLIENT_SHM_TEST_PATH]() { + int raw_fd = shm_open(CLIENT_SHM_TEST_PATH, O_RDWR | O_CREAT, 0700); + if (raw_fd < 0) { + LTTNG_THROW_POSIX( + lttng::format("Failed to open shared memory at path '%s'", + CLIENT_SHM_TEST_PATH), + errno); + } + return lttng::file_descriptor(raw_fd); + }(); + + const auto scope_shm_unlink = lttng::make_scope_exit( + [CLIENT_SHM_TEST_PATH]() noexcept { shm_unlink(CLIENT_SHM_TEST_PATH); }); + if (fstatvfs(fd.fd(), &statbuf) != 0) { + LTTNG_THROW_POSIX( + "Failed to get the capacity of the filesystem at the default location used by shm_open", + errno); + } + } + + memfd_device_size = statbuf.f_frsize * statbuf.f_blocks; + DBG("memfd device id `%lu` has size %lu bytes", statbuf.f_fsid, memfd_device_size); + estimated_size = estimate_session_minimum_shm_size(&sessions[0]); + DBG("Estimated min shm for session '%s': %lu", session_name, estimated_size); + if (estimated_size >= memfd_device_size) { + WARN_FMT( + "The estimated minimum shared memory size for all non-kernel channels of session '{}' is greater than the total shared memory allocated to {} ({}MiB >= {}MiB). Tracing for this session may not record events due to allocation failures.", + session_name, + ret == LTTNG_GET_SESSION_SHM_PATH_STATUS_OK ? + shm_path : + "the default shared memory location", + estimated_size / 1024 / 1024, + memfd_device_size / 1024 / 1024); + } +} +} /* namespace */ + /* * start_tracing * @@ -86,8 +244,13 @@ cmd_error_code start_tracing(const char *session_name) return CMD_ERROR; } - DBG("Starting tracing for session `%s`", session_name); + try { + warn_on_small_client_shm(session_name); + } catch (const lttng::runtime_error& ex) { + DBG("Failed to check client shm size warning: %s", ex.what()); + } + DBG("Starting tracing for session `%s`", session_name); const int ret = lttng_start_tracing(session_name); if (ret < 0) { LTTNG_THROW_CTL(lttng::format("Failed to start session `{}`", session_name), @@ -251,7 +414,6 @@ int cmd_start(int argc, const char **argv) command_ret = CMD_ERROR; goto end; } - /* Open output element */ if (mi_lttng_writer_open_element(writer, mi_lttng_element_command_output)) { command_ret = CMD_ERROR; -- 2.39.5