sessiond: lttng: Add command to check kernel tracer status
authorKienan Stewart <kstewart@efficios.com>
Wed, 22 Nov 2023 21:28:01 +0000 (16:28 -0500)
committerJérémie Galarneau <jeremie.galarneau@efficios.com>
Wed, 28 Feb 2024 15:38:54 +0000 (10:38 -0500)
Issue observed
--------------

When `lttng enable-channel --kernel` fails, little feedback is
available to user to help them to understand the cause.

Eg.

```
Error: Channel asdf: Kernel tracer not available (session auto-20231123-092621)
```

Solution
--------

The semantic status of the kernel tracer is tracked and persisted in
the session daemon (through `init_kernel_tracer` and
`cleanup_tracer_tracer`.

A new client command `lttng_kernel_tracer_status` is added to request
the current value of the `kernel_tracer_status`. The `lttng` client
uses this command after enabling a kernel-domain channel fails to
provide the user with a more specific cause of the failure.

Eg.

```
Error: Channel asdf: Kernel tracer not available (session auto-20231123-092621)
        Missing one or more required kernel modules
        Consult lttng-sessiond logs for more information
```

The kernel tracer status is tracked with an enum defined in
`include/lttng/kernel.h` to avoid passing potentially different errno values
or locale-dependant strings between the LTTng client and session
daemon.

Loading modules and checking signatures can fail with a number of
different errno values. For example:

C.f. https://gitlab.com/linux-kernel/stable/-/blob/master/kernel/module/signing.c#L70

 * `EKEYREJECTED`
 * Any other error code

C.f. https://gitlab.com/linux-kernel/stable/-/blob/master/Documentation/security/keys/core.rst

 * `EKEYREVOKED`
 * `EKEYEXPIRED`
 * `ENOKEY`
 * Others, such as `ENOMEM`

Known drawbacks
---------------

None.

Signed-off-by: Kienan Stewart <kstewart@efficios.com>
Signed-off-by: Jérémie Galarneau <jeremie.galarneau@efficios.com>
Change-Id: I2ae4b188f0110a472200c2511439b9e3e600527d

13 files changed:
include/Makefile.am
include/lttng/kernel.h [new file with mode: 0644]
include/lttng/lttng.h
src/bin/lttng-sessiond/client.cpp
src/bin/lttng-sessiond/cmd.cpp
src/bin/lttng-sessiond/cmd.hpp
src/bin/lttng-sessiond/kernel.cpp
src/bin/lttng-sessiond/kernel.hpp
src/bin/lttng-sessiond/modprobe.cpp
src/bin/lttng/commands/enable_channels.cpp
src/common/sessiond-comm/sessiond-comm.hpp
src/lib/lttng-ctl/liblttng-ctl.sym
src/lib/lttng-ctl/lttng-ctl.cpp

index d840d58dcd8458487348a993feddde279d531485..a2d6a5cf015eea6c9b3b907b4ca917d00b541814 100644 (file)
@@ -110,6 +110,7 @@ lttnginclude_HEADERS = \
        lttng/event.h \
        lttng/handle.h \
        lttng/health.h \
+       lttng/kernel.h \
        lttng/kernel-probe.h \
        lttng/load.h \
        lttng/location.h \
diff --git a/include/lttng/kernel.h b/include/lttng/kernel.h
new file mode 100644 (file)
index 0000000..d3a9f05
--- /dev/null
@@ -0,0 +1,44 @@
+/*
+ * kernel.h
+ *
+ * Linux Trace Toolkit Control Library Header File
+ *
+ * Copyright (C) 2023 Kienan Stewart <kstewart@efficios.com>
+ *
+ * SPDX-License-Identifier; LGPL-2.1-only
+ *
+ */
+
+#ifndef LTTNG_KERNEL_H
+#define LTTNG_KERNEL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum lttng_kernel_tracer_status {
+       /* Loaded without error. */
+       LTTNG_KERNEL_TRACER_STATUS_INITIALIZED = 0,
+       /* Unknown error. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN = -1,
+       /* lttng-sessiond isn't running as root. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT = -2,
+       /* Notifier setup failed. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER = -3,
+       /* Failed to open /proc/lttng. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG = -4,
+       /* Version mismatch between kernel tracer and kernel tracer ABI. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH = -5,
+       /* Kernel module loading failed. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN = -6,
+       /* Kernel modules missing. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING = -7,
+       /* Kernel module signature error. */
+       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE = -8,
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LTTNG_KERNEL_H */
index 929ce014e10b752f133c87589ff492bfb27e7efd..404a80117d50188f4d48491c7e65324862197087 100644 (file)
@@ -53,6 +53,7 @@
 #include <lttng/event.h>
 #include <lttng/handle.h>
 #include <lttng/health.h>
+#include <lttng/kernel.h>
 #include <lttng/kernel-probe.h>
 #include <lttng/load.h>
 #include <lttng/location.h>
@@ -194,6 +195,13 @@ lttng_set_consumer_url(struct lttng_handle *handle, const char *control_url, con
  */
 LTTNG_EXPORT extern int lttng_data_pending(const char *session_name);
 
+/*
+ * Gets the status of the kernel tracer.
+ *
+ * Sets the value of the argument, which must not be null.
+ */
+LTTNG_EXPORT extern enum lttng_error_code lttng_get_kernel_tracer_status(enum lttng_kernel_tracer_status *status);
+
 /*
  * Deprecated, replaced by lttng_regenerate_metadata.
  */
index a89c61e9c54d166a1eb106d202ffba5d9c699775..43a8b793e9ed366d5be2631feca34cc8f190ddef 100644 (file)
@@ -1033,6 +1033,7 @@ static int process_client_msg(struct command_ctx *cmd_ctx, int *sock, int *sock_
        case LTTCOMM_SESSIOND_COMMAND_CLEAR_SESSION:
        case LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS:
        case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY:
+       case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS:
                need_domain = false;
                break;
        default:
@@ -1116,6 +1117,7 @@ static int process_client_msg(struct command_ctx *cmd_ctx, int *sock, int *sock_
        case LTTCOMM_SESSIOND_COMMAND_UNREGISTER_TRIGGER:
        case LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS:
        case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY:
+       case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS:
                need_tracing_session = false;
                break;
        default:
@@ -1927,6 +1929,25 @@ skip_domain:
                        cmd_ctx->session, cmd_ctx->lsm.domain.type, cmd_ctx->lsm.u.reg.path, cdata);
                break;
        }
+       case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS:
+       {
+               uint32_t u_status;
+               enum lttng_kernel_tracer_status status;
+
+               ret = cmd_kernel_tracer_status(&status);
+               if (ret != LTTNG_OK) {
+                       goto error;
+               }
+
+               u_status = (uint32_t) status;
+               ret = setup_lttng_msg_no_cmd_header(cmd_ctx, &u_status, 4);
+               if (ret < 0) {
+                       goto error;
+               }
+
+               ret = LTTNG_OK;
+               break;
+       }
        case LTTCOMM_SESSIOND_COMMAND_DATA_PENDING:
        {
                int pending_ret;
index c1588bc3a37dc3dc1dd9b0facf981a33707b4111..b1e0c098594f26004bdf329567a6b6eb08731dac 100644 (file)
@@ -55,6 +55,7 @@
 #include <lttng/event-internal.hpp>
 #include <lttng/event-rule/event-rule-internal.hpp>
 #include <lttng/event-rule/event-rule.h>
+#include <lttng/kernel.h>
 #include <lttng/location-internal.hpp>
 #include <lttng/lttng-error.h>
 #include <lttng/rotate-internal.hpp>
@@ -4017,6 +4018,19 @@ void cmd_list_lttng_sessions(struct lttng_session *sessions,
        }
 }
 
+/*
+ * Command LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS
+ */
+enum lttng_error_code cmd_kernel_tracer_status(enum lttng_kernel_tracer_status *status)
+{
+       if (status == nullptr) {
+               return LTTNG_ERR_INVALID;
+       }
+
+       *status = get_kernel_tracer_status();
+       return LTTNG_OK;
+}
+
 /*
  * Command LTTNG_DATA_PENDING returning 0 if the data is NOT pending meaning
  * ready for trace analysis (or any kind of reader) or else 1 for pending data.
index 8b77b18a4d7e2f81c24f6a4c8922da2d59b8f8ba..9ae371a3157644f7c5ae9e8fcb99f55c2398c90e 100644 (file)
@@ -14,6 +14,7 @@
 #include "session.hpp"
 
 #include <common/tracker.hpp>
+#include <lttng/kernel.h>
 
 struct notification_thread_handle;
 struct lttng_dynamic_buffer;
@@ -132,6 +133,7 @@ ssize_t cmd_snapshot_list_outputs(struct ltt_session *session,
 enum lttng_error_code cmd_list_syscalls(struct lttng_payload *reply_payload);
 
 int cmd_data_pending(struct ltt_session *session);
+enum lttng_error_code cmd_kernel_tracer_status(enum lttng_kernel_tracer_status *status);
 
 /* Snapshot */
 int cmd_snapshot_add_output(struct ltt_session *session,
index 24c3180d5eabcb05310dfbca4d3ea2207674f91e..ba82045ed86e51c18629efbbfe3a1711cd5a3dac 100644 (file)
@@ -58,6 +58,7 @@ static uint64_t next_kernel_channel_key;
 static const char *module_proc_lttng = "/proc/lttng";
 
 static int kernel_tracer_fd = -1;
+static nonstd::optional<enum lttng_kernel_tracer_status> kernel_tracer_status = nonstd::nullopt;
 static int kernel_tracer_event_notifier_group_fd = -1;
 static int kernel_tracer_event_notifier_group_notification_fd = -1;
 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
@@ -1949,6 +1950,48 @@ error:
        return ret;
 }
 
+/*
+ * Get current kernel tracer status
+ */
+enum lttng_kernel_tracer_status get_kernel_tracer_status()
+{
+       if (!kernel_tracer_status) {
+               return LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN;
+       }
+
+       return *kernel_tracer_status;
+}
+
+/*
+ * Sets the kernel tracer status based on the positive errno code
+ */
+void set_kernel_tracer_status_from_modules_ret(int code)
+{
+       switch (code) {
+       case ENOENT:
+       {
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING);
+               break;
+       }
+       case ENOKEY:
+       case EKEYEXPIRED:
+       case EKEYREVOKED:
+       case EKEYREJECTED:
+       {
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE);
+               break;
+       }
+       default:
+       {
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN);
+               break;
+       }
+       }
+}
+
 /*
  * Setup necessary data for kernel tracer action.
  */
@@ -1960,6 +2003,7 @@ int init_kernel_tracer()
        /* Modprobe lttng kernel modules */
        ret = modprobe_lttng_control();
        if (ret < 0) {
+               set_kernel_tracer_status_from_modules_ret(-ret);
                goto error;
        }
 
@@ -1967,17 +2011,22 @@ int init_kernel_tracer()
        kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
        if (kernel_tracer_fd < 0) {
                DBG("Failed to open %s", module_proc_lttng);
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG);
                goto error_open;
        }
 
        /* Validate kernel version */
        ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
        if (ret < 0) {
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH);
                goto error_version;
        }
 
        ret = modprobe_lttng_data();
        if (ret < 0) {
+               set_kernel_tracer_status_from_modules_ret(-ret);
                goto error_modules;
        }
 
@@ -1994,6 +2043,8 @@ int init_kernel_tracer()
        ret = kernel_supports_event_notifiers();
        if (ret < 0) {
                ERR("Failed to check for kernel tracer event notifier support");
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
                goto error_modules;
        }
        ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
@@ -2008,6 +2059,8 @@ int init_kernel_tracer()
                                &kernel_tracer_event_notifier_group_notification_fd);
 
                if (error_code_ret != LTTNG_OK) {
+                       kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                               LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
                        goto error_modules;
                }
 
@@ -2016,12 +2069,16 @@ int init_kernel_tracer()
                if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
                        ERR("Failed to initialize event notifier error accounting for kernel tracer");
                        error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
+                       kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                               LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
                        goto error_modules;
                }
 
                kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
                        DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
                if (!kernel_token_to_event_notifier_rule_ht) {
+                       kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                               LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
                        goto error_token_ht;
                }
        }
@@ -2037,6 +2094,7 @@ int init_kernel_tracer()
                    "work for this session daemon.");
        }
 
+       kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(LTTNG_KERNEL_TRACER_STATUS_INITIALIZED);
        return 0;
 
 error_version:
@@ -2081,6 +2139,8 @@ error:
        WARN("No kernel tracer available");
        kernel_tracer_fd = -1;
        if (!is_root) {
+               kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
+                       LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT);
                return LTTNG_ERR_NEED_ROOT_SESSIOND;
        } else {
                return LTTNG_ERR_KERN_NA;
@@ -2136,6 +2196,7 @@ void cleanup_kernel_tracer()
                kernel_tracer_fd = -1;
        }
 
+       kernel_tracer_status = nonstd::nullopt;
        free(syscall_table);
 }
 
index 6024f611c8005627c9e1df4f07b4f5f7c2747509..bbcbcf9a4f65afa13af6d4dd8defc0f7b2686ea1 100644 (file)
@@ -74,6 +74,8 @@ int init_kernel_workarounds(void);
 int kernel_supports_ring_buffer_snapshot_sample_positions(void);
 int kernel_supports_ring_buffer_packet_sequence_number(void);
 int kernel_supports_event_notifiers(void);
+enum lttng_kernel_tracer_status get_kernel_tracer_status(void);
+void set_kernel_tracer_status_from_modules_ret(int);
 int init_kernel_tracer(void);
 void cleanup_kernel_tracer(void);
 bool kernel_tracer_is_initialized(void);
index 547756de9413d864eb908b351ec9d777c2702364..0b274c9a3fae6e4986f4704c9bdc5e7c4ba221f7 100644 (file)
@@ -540,6 +540,10 @@ static int modprobe_lttng(struct kern_modules_param *modules, int entries)
                } else if (WEXITSTATUS(ret) != 0) {
                        if (modules[i].load_policy == KERNEL_MODULE_PROPERTY_LOAD_POLICY_REQUIRED) {
                                ERR("Unable to load required module %s", modules[i].name);
+                               /* Force negative return code */
+                               if (ret > 0) {
+                                       ret = -ret;
+                               }
                                goto error;
                        } else {
                                DBG("Unable to load optional module %s; continuing",
index c302132fbca258e4e5234c679ef4073a38612092..1638d0d446b88108aa2d96e595f622a2831e6298 100644 (file)
@@ -11,6 +11,7 @@
 
 #include <common/mi-lttng.hpp>
 #include <common/sessiond-comm/sessiond-comm.hpp>
+#include <common/lttng-kernel.hpp>
 #include <common/utils.hpp>
 
 #include <lttng/domain-internal.hpp>
@@ -141,6 +142,7 @@ static int enable_channel(char *session_name, char *channel_list)
 {
        struct lttng_channel *channel = nullptr;
        int ret = CMD_SUCCESS, warn = 0, error = 0, success = 0;
+       enum lttng_kernel_tracer_status kernel_tracer_status;
        char *channel_name;
        struct lttng_domain dom;
 
@@ -309,6 +311,43 @@ static int enable_channel(char *session_name, char *channel_list)
                                error = 1;
                                break;
                        }
+                       /*
+                        * Ask the sessiond for the more details on the status of the kernel tracer.
+                        */
+                       ret = lttng_get_kernel_tracer_status(&kernel_tracer_status);
+                       if (ret < 0) {
+                               ERR("Failed to get kernel tracer status: %s", lttng_strerror(ret));
+                       } else {
+                               switch (kernel_tracer_status) {
+                               case LTTNG_KERNEL_TRACER_STATUS_INITIALIZED:
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN:
+                                       MSG("\tKernel module loading failed");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING:
+                                       MSG("\tMissing one or more required kernel modules");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE:
+                                       MSG("\tKernel module signature error prevented loading of one or more required kernel modules");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT:
+                                       MSG("\tlttng-sessiond isn't running as root");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER:
+                                       MSG("\tFailed to setup notifiers");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG:
+                                       MSG("\tlttng-sessiond failed to open proc lttng");
+                                       break;
+                               case LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH:
+                                       MSG("\tVersion mismatch between kernel tracer and kernel tracer ABI");
+                                       break;
+                               default:
+                                       MSG("\tUnknown kernel tracer status (%d)", kernel_tracer_status);
+                                       break;
+                               }
+                               MSG("\tConsult lttng-sessiond logs for more information");
+                       }
                } else {
                        MSG("%s channel %s enabled for session %s",
                            lttng_domain_type_str(dom.type),
index 982d517f1ad5a63d87ff7f8dce58a44290e89059..c8ac0cf41e60318b4634bf2ee4f25b1637ae6680 100644 (file)
@@ -95,6 +95,7 @@ enum lttcomm_sessiond_command {
        LTTCOMM_SESSIOND_COMMAND_CLEAR_SESSION,
        LTTCOMM_SESSIOND_COMMAND_LIST_TRIGGERS,
        LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY,
+       LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS,
        LTTCOMM_SESSIOND_COMMAND_MAX,
 };
 
@@ -192,6 +193,8 @@ static inline const char *lttcomm_sessiond_command_str(enum lttcomm_sessiond_com
                return "LIST_TRIGGERS";
        case LTTCOMM_SESSIOND_COMMAND_EXECUTE_ERROR_QUERY:
                return "EXECUTE_ERROR_QUERY";
+       case LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS:
+               return "KERNEL_TRACER_STATUS";
        default:
                abort();
        }
index b69e78df9cb8b45d66eb4c20a12837a99c742ceb..3bd9efb76cc2c64f895d1205ae4c29b0005fb899 100644 (file)
@@ -311,6 +311,7 @@ lttng_event_rule_user_tracepoint_set_filter
 lttng_event_rule_user_tracepoint_set_log_level_rule
 lttng_event_rule_user_tracepoint_set_name_pattern
 lttng_event_set_userspace_probe_location
+lttng_get_kernel_tracer_status
 lttng_health_create_consumerd
 lttng_health_create_relayd
 lttng_health_create_sessiond
index 15a699e24f3fd63837edbc6b11dccfa3ac5b1a10..22823d344bd1d556a3248ab63c0f093cd7cd7ac7 100644 (file)
@@ -2877,6 +2877,35 @@ end:
        return ret;
 }
 
+/*
+ * Get the status of the kernel tracer
+ *
+ * Sets the value of the argument
+ */
+enum lttng_error_code lttng_get_kernel_tracer_status(enum lttng_kernel_tracer_status *status)
+{
+       enum lttng_error_code ret = LTTNG_ERR_INVALID;
+
+       if (status == nullptr) {
+               return LTTNG_ERR_INVALID;
+       }
+
+       struct lttcomm_session_msg lsm = {};
+       lsm.cmd_type = LTTCOMM_SESSIOND_COMMAND_KERNEL_TRACER_STATUS;
+
+       uint32_t *u_status = nullptr;
+       const auto ask_ret = lttng_ctl_ask_sessiond(&lsm, (void **) &u_status);
+       if (ask_ret != 4) {
+               goto end;
+       }
+
+       *status = (enum lttng_kernel_tracer_status) * u_status;
+       ret = LTTNG_OK;
+end:
+       free(u_status);
+       return ret;
+}
+
 /*
  * Regenerate the metadata for a session.
  * Return 0 on success, a negative error code on error.
This page took 0.043561 seconds and 4 git commands to generate.