From 49cddecdbb1e042877ca3855b177ade68151030f Mon Sep 17 00:00:00 2001 From: Kienan Stewart Date: Wed, 22 Nov 2023 16:28:01 -0500 Subject: [PATCH] sessiond: lttng: Add command to check kernel tracer status MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Issue observed -------------- When `lttng enable-channel --kernel` fails, little feedback is available to user to help them to understand the cause. Eg. ``` Error: Channel asdf: Kernel tracer not available (session auto-20231123-092621) ``` Solution -------- The semantic status of the kernel tracer is tracked and persisted in the session daemon (through `init_kernel_tracer` and `cleanup_tracer_tracer`. A new client command `lttng_kernel_tracer_status` is added to request the current value of the `kernel_tracer_status`. The `lttng` client uses this command after enabling a kernel-domain channel fails to provide the user with a more specific cause of the failure. Eg. ``` Error: Channel asdf: Kernel tracer not available (session auto-20231123-092621) Missing one or more required kernel modules Consult lttng-sessiond logs for more information ``` The kernel tracer status is tracked with an enum defined in `include/lttng/kernel.h` to avoid passing potentially different errno values or locale-dependant strings between the LTTng client and session daemon. Loading modules and checking signatures can fail with a number of different errno values. For example: C.f. https://gitlab.com/linux-kernel/stable/-/blob/master/kernel/module/signing.c#L70 * `EKEYREJECTED` * Any other error code C.f. https://gitlab.com/linux-kernel/stable/-/blob/master/Documentation/security/keys/core.rst * `EKEYREVOKED` * `EKEYEXPIRED` * `ENOKEY` * Others, such as `ENOMEM` Known drawbacks --------------- None. Signed-off-by: Kienan Stewart Signed-off-by: Jérémie Galarneau Change-Id: I2ae4b188f0110a472200c2511439b9e3e600527d --- include/Makefile.am | 1 + include/lttng/kernel.h | 44 ++++++++++++++++ include/lttng/lttng.h | 8 +++ src/bin/lttng-sessiond/client.cpp | 21 ++++++++ src/bin/lttng-sessiond/cmd.cpp | 14 +++++ src/bin/lttng-sessiond/cmd.hpp | 2 + src/bin/lttng-sessiond/kernel.cpp | 61 ++++++++++++++++++++++ src/bin/lttng-sessiond/kernel.hpp | 2 + src/bin/lttng-sessiond/modprobe.cpp | 4 ++ src/bin/lttng/commands/enable_channels.cpp | 39 ++++++++++++++ src/common/sessiond-comm/sessiond-comm.hpp | 3 ++ src/lib/lttng-ctl/liblttng-ctl.sym | 1 + src/lib/lttng-ctl/lttng-ctl.cpp | 29 ++++++++++ 13 files changed, 229 insertions(+) create mode 100644 include/lttng/kernel.h diff --git a/include/Makefile.am b/include/Makefile.am index d840d58dc..a2d6a5cf0 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -110,6 +110,7 @@ lttnginclude_HEADERS = \ lttng/event.h \ lttng/handle.h \ lttng/health.h \ + lttng/kernel.h \ lttng/kernel-probe.h \ lttng/load.h \ lttng/location.h \ diff --git a/include/lttng/kernel.h b/include/lttng/kernel.h new file mode 100644 index 000000000..d3a9f05b5 --- /dev/null +++ b/include/lttng/kernel.h @@ -0,0 +1,44 @@ +/* + * kernel.h + * + * Linux Trace Toolkit Control Library Header File + * + * Copyright (C) 2023 Kienan Stewart + * + * SPDX-License-Identifier; LGPL-2.1-only + * + */ + +#ifndef LTTNG_KERNEL_H +#define LTTNG_KERNEL_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum lttng_kernel_tracer_status { + /* Loaded without error. */ + LTTNG_KERNEL_TRACER_STATUS_INITIALIZED = 0, + /* Unknown error. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN = -1, + /* lttng-sessiond isn't running as root. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT = -2, + /* Notifier setup failed. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER = -3, + /* Failed to open /proc/lttng. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG = -4, + /* Version mismatch between kernel tracer and kernel tracer ABI. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH = -5, + /* Kernel module loading failed. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN = -6, + /* Kernel modules missing. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING = -7, + /* Kernel module signature error. */ + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE = -8, +}; + +#ifdef __cplusplus +} +#endif + +#endif /* LTTNG_KERNEL_H */ diff --git a/include/lttng/lttng.h b/include/lttng/lttng.h index 929ce014e..404a80117 100644 --- a/include/lttng/lttng.h +++ b/include/lttng/lttng.h @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -194,6 +195,13 @@ lttng_set_consumer_url(struct lttng_handle *handle, const char *control_url, con */ LTTNG_EXPORT extern int lttng_data_pending(const char *session_name); +/* + * Gets the status of the kernel tracer. + * + * Sets the value of the argument, which must not be null. + */ +LTTNG_EXPORT extern enum lttng_error_code lttng_get_kernel_tracer_status(enum lttng_kernel_tracer_status *status); + /* * Deprecated, replaced by lttng_regenerate_metadata. */ diff --git a/src/bin/lttng-sessiond/client.cpp b/src/bin/lttng-sessiond/client.cpp index a89c61e9c..43a8b793e 100644 --- a/src/bin/lttng-sessiond/client.cpp +++ b/src/bin/lttng-sessiond/client.cpp @@ -1033,6 +1033,7 @@ static int process_client_msg(struct command_ctx *cmd_ctx, int *sock, int *sock_ case LTTCOMM_SESSIOND_COMMAND_CLEAR_SESSION: case LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS: case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY: + case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS: need_domain = false; break; default: @@ -1116,6 +1117,7 @@ static int process_client_msg(struct command_ctx *cmd_ctx, int *sock, int *sock_ case LTTCOMM_SESSIOND_COMMAND_UNREGISTER_TRIGGER: case LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS: case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY: + case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS: need_tracing_session = false; break; default: @@ -1927,6 +1929,25 @@ skip_domain: cmd_ctx->session, cmd_ctx->lsm.domain.type, cmd_ctx->lsm.u.reg.path, cdata); break; } + case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS: + { + uint32_t u_status; + enum lttng_kernel_tracer_status status; + + ret = cmd_kernel_tracer_status(&status); + if (ret != LTTNG_OK) { + goto error; + } + + u_status = (uint32_t) status; + ret = setup_lttng_msg_no_cmd_header(cmd_ctx, &u_status, 4); + if (ret < 0) { + goto error; + } + + ret = LTTNG_OK; + break; + } case LTTCOMM_SESSIOND_COMMAND_DATA_PENDING: { int pending_ret; diff --git a/src/bin/lttng-sessiond/cmd.cpp b/src/bin/lttng-sessiond/cmd.cpp index c1588bc3a..b1e0c0985 100644 --- a/src/bin/lttng-sessiond/cmd.cpp +++ b/src/bin/lttng-sessiond/cmd.cpp @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -4017,6 +4018,19 @@ void cmd_list_lttng_sessions(struct lttng_session *sessions, } } +/* + * Command LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS + */ +enum lttng_error_code cmd_kernel_tracer_status(enum lttng_kernel_tracer_status *status) +{ + if (status == nullptr) { + return LTTNG_ERR_INVALID; + } + + *status = get_kernel_tracer_status(); + return LTTNG_OK; +} + /* * Command LTTNG_DATA_PENDING returning 0 if the data is NOT pending meaning * ready for trace analysis (or any kind of reader) or else 1 for pending data. diff --git a/src/bin/lttng-sessiond/cmd.hpp b/src/bin/lttng-sessiond/cmd.hpp index 8b77b18a4..9ae371a31 100644 --- a/src/bin/lttng-sessiond/cmd.hpp +++ b/src/bin/lttng-sessiond/cmd.hpp @@ -14,6 +14,7 @@ #include "session.hpp" #include +#include struct notification_thread_handle; struct lttng_dynamic_buffer; @@ -132,6 +133,7 @@ ssize_t cmd_snapshot_list_outputs(struct ltt_session *session, enum lttng_error_code cmd_list_syscalls(struct lttng_payload *reply_payload); int cmd_data_pending(struct ltt_session *session); +enum lttng_error_code cmd_kernel_tracer_status(enum lttng_kernel_tracer_status *status); /* Snapshot */ int cmd_snapshot_add_output(struct ltt_session *session, diff --git a/src/bin/lttng-sessiond/kernel.cpp b/src/bin/lttng-sessiond/kernel.cpp index 24c3180d5..ba82045ed 100644 --- a/src/bin/lttng-sessiond/kernel.cpp +++ b/src/bin/lttng-sessiond/kernel.cpp @@ -58,6 +58,7 @@ static uint64_t next_kernel_channel_key; static const char *module_proc_lttng = "/proc/lttng"; static int kernel_tracer_fd = -1; +static nonstd::optional kernel_tracer_status = nonstd::nullopt; static int kernel_tracer_event_notifier_group_fd = -1; static int kernel_tracer_event_notifier_group_notification_fd = -1; static struct cds_lfht *kernel_token_to_event_notifier_rule_ht; @@ -1949,6 +1950,48 @@ error: return ret; } +/* + * Get current kernel tracer status + */ +enum lttng_kernel_tracer_status get_kernel_tracer_status() +{ + if (!kernel_tracer_status) { + return LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN; + } + + return *kernel_tracer_status; +} + +/* + * Sets the kernel tracer status based on the positive errno code + */ +void set_kernel_tracer_status_from_modules_ret(int code) +{ + switch (code) { + case ENOENT: + { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING); + break; + } + case ENOKEY: + case EKEYEXPIRED: + case EKEYREVOKED: + case EKEYREJECTED: + { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE); + break; + } + default: + { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN); + break; + } + } +} + /* * Setup necessary data for kernel tracer action. */ @@ -1960,6 +2003,7 @@ int init_kernel_tracer() /* Modprobe lttng kernel modules */ ret = modprobe_lttng_control(); if (ret < 0) { + set_kernel_tracer_status_from_modules_ret(-ret); goto error; } @@ -1967,17 +2011,22 @@ int init_kernel_tracer() kernel_tracer_fd = open(module_proc_lttng, O_RDWR); if (kernel_tracer_fd < 0) { DBG("Failed to open %s", module_proc_lttng); + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG); goto error_open; } /* Validate kernel version */ ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version); if (ret < 0) { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH); goto error_version; } ret = modprobe_lttng_data(); if (ret < 0) { + set_kernel_tracer_status_from_modules_ret(-ret); goto error_modules; } @@ -1994,6 +2043,8 @@ int init_kernel_tracer() ret = kernel_supports_event_notifiers(); if (ret < 0) { ERR("Failed to check for kernel tracer event notifier support"); + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER); goto error_modules; } ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd); @@ -2008,6 +2059,8 @@ int init_kernel_tracer() &kernel_tracer_event_notifier_group_notification_fd); if (error_code_ret != LTTNG_OK) { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER); goto error_modules; } @@ -2016,12 +2069,16 @@ int init_kernel_tracer() if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) { ERR("Failed to initialize event notifier error accounting for kernel tracer"); error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING; + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER); goto error_modules; } kernel_token_to_event_notifier_rule_ht = cds_lfht_new( DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr); if (!kernel_token_to_event_notifier_rule_ht) { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER); goto error_token_ht; } } @@ -2037,6 +2094,7 @@ int init_kernel_tracer() "work for this session daemon."); } + kernel_tracer_status = nonstd::optional(LTTNG_KERNEL_TRACER_STATUS_INITIALIZED); return 0; error_version: @@ -2081,6 +2139,8 @@ error: WARN("No kernel tracer available"); kernel_tracer_fd = -1; if (!is_root) { + kernel_tracer_status = nonstd::optional( + LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT); return LTTNG_ERR_NEED_ROOT_SESSIOND; } else { return LTTNG_ERR_KERN_NA; @@ -2136,6 +2196,7 @@ void cleanup_kernel_tracer() kernel_tracer_fd = -1; } + kernel_tracer_status = nonstd::nullopt; free(syscall_table); } diff --git a/src/bin/lttng-sessiond/kernel.hpp b/src/bin/lttng-sessiond/kernel.hpp index 6024f611c..bbcbcf9a4 100644 --- a/src/bin/lttng-sessiond/kernel.hpp +++ b/src/bin/lttng-sessiond/kernel.hpp @@ -74,6 +74,8 @@ int init_kernel_workarounds(void); int kernel_supports_ring_buffer_snapshot_sample_positions(void); int kernel_supports_ring_buffer_packet_sequence_number(void); int kernel_supports_event_notifiers(void); +enum lttng_kernel_tracer_status get_kernel_tracer_status(void); +void set_kernel_tracer_status_from_modules_ret(int); int init_kernel_tracer(void); void cleanup_kernel_tracer(void); bool kernel_tracer_is_initialized(void); diff --git a/src/bin/lttng-sessiond/modprobe.cpp b/src/bin/lttng-sessiond/modprobe.cpp index 547756de9..0b274c9a3 100644 --- a/src/bin/lttng-sessiond/modprobe.cpp +++ b/src/bin/lttng-sessiond/modprobe.cpp @@ -540,6 +540,10 @@ static int modprobe_lttng(struct kern_modules_param *modules, int entries) } else if (WEXITSTATUS(ret) != 0) { if (modules[i].load_policy == KERNEL_MODULE_PROPERTY_LOAD_POLICY_REQUIRED) { ERR("Unable to load required module %s", modules[i].name); + /* Force negative return code */ + if (ret > 0) { + ret = -ret; + } goto error; } else { DBG("Unable to load optional module %s; continuing", diff --git a/src/bin/lttng/commands/enable_channels.cpp b/src/bin/lttng/commands/enable_channels.cpp index c302132fb..1638d0d44 100644 --- a/src/bin/lttng/commands/enable_channels.cpp +++ b/src/bin/lttng/commands/enable_channels.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -141,6 +142,7 @@ static int enable_channel(char *session_name, char *channel_list) { struct lttng_channel *channel = nullptr; int ret = CMD_SUCCESS, warn = 0, error = 0, success = 0; + enum lttng_kernel_tracer_status kernel_tracer_status; char *channel_name; struct lttng_domain dom; @@ -309,6 +311,43 @@ static int enable_channel(char *session_name, char *channel_list) error = 1; break; } + /* + * Ask the sessiond for the more details on the status of the kernel tracer. + */ + ret = lttng_get_kernel_tracer_status(&kernel_tracer_status); + if (ret < 0) { + ERR("Failed to get kernel tracer status: %s", lttng_strerror(ret)); + } else { + switch (kernel_tracer_status) { + case LTTNG_KERNEL_TRACER_STATUS_INITIALIZED: + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN: + MSG("\tKernel module loading failed"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING: + MSG("\tMissing one or more required kernel modules"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE: + MSG("\tKernel module signature error prevented loading of one or more required kernel modules"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT: + MSG("\tlttng-sessiond isn't running as root"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER: + MSG("\tFailed to setup notifiers"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG: + MSG("\tlttng-sessiond failed to open proc lttng"); + break; + case LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH: + MSG("\tVersion mismatch between kernel tracer and kernel tracer ABI"); + break; + default: + MSG("\tUnknown kernel tracer status (%d)", kernel_tracer_status); + break; + } + MSG("\tConsult lttng-sessiond logs for more information"); + } } else { MSG("%s channel %s enabled for session %s", lttng_domain_type_str(dom.type), diff --git a/src/common/sessiond-comm/sessiond-comm.hpp b/src/common/sessiond-comm/sessiond-comm.hpp index 982d517f1..c8ac0cf41 100644 --- a/src/common/sessiond-comm/sessiond-comm.hpp +++ b/src/common/sessiond-comm/sessiond-comm.hpp @@ -95,6 +95,7 @@ enum lttcomm_sessiond_command { LTTCOMM_SESSIOND_COMMAND_CLEAR_SESSION, LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS, LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY, + LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS, LTTCOMM_SESSIOND_COMMAND_MAX, }; @@ -192,6 +193,8 @@ static inline const char *lttcomm_sessiond_command_str(enum lttcomm_sessiond_com return "LIST_TRIGGERS"; case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY: return "EXECUTE_ERROR_QUERY"; + case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS: + return "KERNEL_TRACER_STATUS"; default: abort(); } diff --git a/src/lib/lttng-ctl/liblttng-ctl.sym b/src/lib/lttng-ctl/liblttng-ctl.sym index b69e78df9..3bd9efb76 100644 --- a/src/lib/lttng-ctl/liblttng-ctl.sym +++ b/src/lib/lttng-ctl/liblttng-ctl.sym @@ -311,6 +311,7 @@ lttng_event_rule_user_tracepoint_set_filter lttng_event_rule_user_tracepoint_set_log_level_rule lttng_event_rule_user_tracepoint_set_name_pattern lttng_event_set_userspace_probe_location +lttng_get_kernel_tracer_status lttng_health_create_consumerd lttng_health_create_relayd lttng_health_create_sessiond diff --git a/src/lib/lttng-ctl/lttng-ctl.cpp b/src/lib/lttng-ctl/lttng-ctl.cpp index 15a699e24..22823d344 100644 --- a/src/lib/lttng-ctl/lttng-ctl.cpp +++ b/src/lib/lttng-ctl/lttng-ctl.cpp @@ -2877,6 +2877,35 @@ end: return ret; } +/* + * Get the status of the kernel tracer + * + * Sets the value of the argument + */ +enum lttng_error_code lttng_get_kernel_tracer_status(enum lttng_kernel_tracer_status *status) +{ + enum lttng_error_code ret = LTTNG_ERR_INVALID; + + if (status == nullptr) { + return LTTNG_ERR_INVALID; + } + + struct lttcomm_session_msg lsm = {}; + lsm.cmd_type = LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS; + + uint32_t *u_status = nullptr; + const auto ask_ret = lttng_ctl_ask_sessiond(&lsm, (void **) &u_status); + if (ask_ret != 4) { + goto end; + } + + *status = (enum lttng_kernel_tracer_status) * u_status; + ret = LTTNG_OK; +end: + free(u_status); + return ret; +} + /* * Regenerate the metadata for a session. * Return 0 on success, a negative error code on error. -- 2.34.1