From 99414eafc01b29540318e5cfdad45c271816140f Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 3 Mar 2022 12:12:44 -0500 Subject: [PATCH] kretprobe: Implement kretprobes with event enablers This enables support for enabling only entry or exit for kretprobes. This enables event notifier support for kretprobes. Signed-off-by: Mathieu Desnoyers Change-Id: Ieef4f2744d5e1ebfc2f3320862e143bfe6401620 --- include/lttng/events-internal.h | 74 +++++---- src/lttng-abi.c | 149 ++---------------- src/lttng-events.c | 176 ++++++++++++++------- src/lttng-kretprobes.c | 268 +++++++++++++++----------------- 4 files changed, 305 insertions(+), 362 deletions(-) diff --git a/include/lttng/events-internal.h b/include/lttng/events-internal.h index eda1a7b2..0bf5b75b 100644 --- a/include/lttng/events-internal.h +++ b/include/lttng/events-internal.h @@ -109,6 +109,13 @@ enum lttng_syscall_abi { LTTNG_SYSCALL_ABI_COMPAT, }; +enum lttng_kretprobe_entryexit { + LTTNG_KRETPROBE_ENTRY, + LTTNG_KRETPROBE_EXIT, + + NR_LTTNG_KRETPROBE_ENTRYEXIT, +}; + struct lttng_kernel_event_common_private { struct lttng_kernel_event_common *pub; /* Public event interface */ @@ -131,6 +138,7 @@ struct lttng_kernel_event_common_private { struct lttng_kprobe kprobe; struct lttng_uprobe uprobe; struct { + enum lttng_kretprobe_entryexit entryexit; struct lttng_krp *lttng_krp; char *symbol_name; } kretprobe; @@ -461,10 +469,9 @@ struct lttng_metadata_stream { struct lttng_kernel_event_pair { /* Input */ char name[LTTNG_KERNEL_ABI_SYM_NAME_LEN]; - - /* Output from lttng_kernel_event_create() */ - struct lttng_kernel_event_common *event[2]; - int refcount; + struct lttng_krp *krp; + bool check_ids; + enum lttng_kretprobe_entryexit entryexit; }; struct lttng_kernel_channel_buffer_ops_private { @@ -1108,53 +1115,62 @@ void lttng_uprobes_destroy_event_private(struct lttng_kernel_event_common *event #endif #ifdef CONFIG_KRETPROBES -struct lttng_kernel_event_desc *lttng_create_kretprobes_event_desc(const char *name); -int lttng_kretprobes_register(const char *symbol_name, - uint64_t offset, - uint64_t addr, - struct lttng_kernel_event_common *event_entry, - struct lttng_kernel_event_common *event_exit); -void lttng_kretprobes_unregister(struct lttng_kernel_event_common *event); -void lttng_kretprobes_destroy_private(struct lttng_kernel_event_common *event); -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, - int enable); +struct lttng_krp *lttng_kretprobes_create_krp(const char *symbol_name, + uint64_t offset, uint64_t addr); +void lttng_kretprobes_put_krp(struct lttng_krp *krp); +int lttng_kretprobes_init_event(const char *name, + enum lttng_kretprobe_entryexit entryexit, + struct lttng_kernel_event_common *event, + struct lttng_krp *krp); +int lttng_kretprobes_register_event(struct lttng_kernel_event_common *event); +void lttng_kretprobes_unregister_event(struct lttng_kernel_event_common *event); +void lttng_kretprobes_destroy_event_private(struct lttng_kernel_event_common *event); int lttng_kretprobes_match_check(const char *symbol_name, uint64_t offset, uint64_t addr); #else static inline -struct lttng_kernel_event_desc *lttng_create_kretprobes_event_desc(const char *name) +struct lttng_krp *lttng_kretprobes_create_krp(const char *symbol_name, + uint64_t offset, uint64_t addr) { return NULL; } static inline -int lttng_kretprobes_register(const char *symbol_name, - uint64_t offset, - uint64_t addr, - struct lttng_kernel_event_common *event_entry, - struct lttng_kernel_event_common *event_exit) +void lttng_kretprobes_put_krp(struct lttng_krp *krp) { } +int lttng_kretprobes_init_event(const char *name, + enum lttng_kretprobe_entryexit entryexit, + struct lttng_kernel_event_common *event, + struct lttng_krp *krp) { return -ENOSYS; } - static inline -void lttng_kretprobes_unregister(struct lttng_kernel_event_common *event) +int lttng_kretprobes_register_event(struct lttng_kernel_event_common *event) { + return -ENOSYS; } - static inline -void lttng_kretprobes_destroy_private(struct lttng_kernel_event_common *event) +void lttng_kretprobes_unregister_event(struct lttng_kernel_event_common *event) { } +static inline +void lttng_kretprobes_destroy_event_private(struct lttng_kernel_event_common *event) { } +static inline +int lttng_kretprobes_match_check(const char *symbol_name, uint64_t offset, uint64_t addr) { + return -ENOENT; } + static inline -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, - int enable) +struct lttng_kernel_event_desc *lttng_create_kretprobes_event_desc(const char *name) { - return -ENOSYS; + return NULL; } static inline -int lttng_kretprobes_match_check(const char *symbol_name, uint64_t offset, uint64_t addr) +int lttng_kretprobes_register(const char *symbol_name, + uint64_t offset, + uint64_t addr, + struct lttng_kernel_event_common *event_entry, + struct lttng_kernel_event_common *event_exit) { - return -ENOENT; + return -ENOSYS; } #endif diff --git a/src/lttng-abi.c b/src/lttng-abi.c index e5b03393..ec8727d3 100644 --- a/src/lttng-abi.c +++ b/src/lttng-abi.c @@ -1963,11 +1963,11 @@ int lttng_abi_validate_event_param(struct lttng_kernel_abi_event *event_param, case LTTNG_KERNEL_ABI_KRETPROBE: switch (event_param->u.kretprobe.entryexit) { case LTTNG_KERNEL_ABI_KRETPROBE_ENTRYEXIT: - break; + lttng_fallthrough; case LTTNG_KERNEL_ABI_KRETPROBE_ENTRY: lttng_fallthrough; case LTTNG_KERNEL_ABI_KRETPROBE_EXIT: - lttng_fallthrough; + break; default: return -EINVAL; } @@ -2121,10 +2121,11 @@ int lttng_abi_create_event_recorder_enabler(struct file *channel_file, case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: fops = <tng_event_session_enabler_fops; break; - case LTTNG_KERNEL_ABI_KRETPROBE: - lttng_fallthrough; + case LTTNG_KERNEL_ABI_UPROBE: fops = <tng_event_session_fops; break; @@ -2185,6 +2186,8 @@ int lttng_abi_create_event_recorder_enabler(struct file *channel_file, } case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: { struct lttng_event_recorder_enabler *event_enabler; @@ -2221,48 +2224,6 @@ int lttng_abi_create_event_recorder_enabler(struct file *channel_file, break; } - case LTTNG_KERNEL_ABI_KRETPROBE: - { - struct lttng_kernel_event_common *event[2]; - struct lttng_event_recorder_enabler *event_enabler; - struct lttng_kernel_event_pair event_pair; - - if (strlen(event_param->name) + strlen("_entry") >= LTTNG_KERNEL_ABI_SYM_NAME_LEN) - return -EINVAL; - - memset(&event_pair, 0, sizeof(event_pair)); - event_enabler = lttng_event_recorder_enabler_create(LTTNG_ENABLER_FORMAT_NAME, - event_param, channel); - if (!event_enabler) { - ret = -ENOMEM; - goto event_error; - } - - strcpy(event_pair.name, event_param->name); - strcat(event_pair.name, "_entry"); - /* - * We tolerate no failure path after event creation. It - * will stay invariant for the rest of the session. - */ - event[0] = lttng_kernel_event_create(&event_enabler->parent.parent, NULL, &event_pair); - if (IS_ERR(event[0])) { - lttng_event_enabler_destroy(&event_enabler->parent.parent); - ret = PTR_ERR(event[0]); - goto event_error; - } - - strcpy(event_pair.name, event_param->name); - strcat(event_pair.name, "_exit"); - event[1] = lttng_kernel_event_create(&event_enabler->parent.parent, NULL, &event_pair); - lttng_event_enabler_destroy(&event_enabler->parent.parent); - if (IS_ERR(event[1])) { - ret = PTR_ERR(event[1]); - goto event_error; - } - priv = event[0]; - break; - } - case LTTNG_KERNEL_ABI_FUNCTION: lttng_fallthrough; case LTTNG_KERNEL_ABI_NOOP: @@ -2319,11 +2280,11 @@ int lttng_abi_create_event_counter_enabler(struct file *channel_file, case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: fops = <tng_event_session_enabler_fops; break; - case LTTNG_KERNEL_ABI_KRETPROBE: - lttng_fallthrough; case LTTNG_KERNEL_ABI_UPROBE: fops = <tng_event_session_fops; break; @@ -2384,6 +2345,8 @@ int lttng_abi_create_event_counter_enabler(struct file *channel_file, } case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: { struct lttng_event_counter_enabler *event_enabler; @@ -2420,48 +2383,6 @@ int lttng_abi_create_event_counter_enabler(struct file *channel_file, break; } - case LTTNG_KERNEL_ABI_KRETPROBE: - { - struct lttng_kernel_event_common *event[2]; - struct lttng_event_counter_enabler *event_enabler; - struct lttng_kernel_event_pair event_pair; - - if (strlen(event_param->name) + strlen("_entry") >= LTTNG_KERNEL_ABI_SYM_NAME_LEN) - return -EINVAL; - - memset(&event_pair, 0, sizeof(event_pair)); - event_enabler = lttng_event_counter_enabler_create(LTTNG_ENABLER_FORMAT_NAME, - event_param, key_param, NULL, channel); - if (!event_enabler) { - ret = -ENOMEM; - goto event_error; - } - - strcpy(event_pair.name, event_param->name); - strcat(event_pair.name, "_entry"); - /* - * We tolerate no failure path after event creation. It - * will stay invariant for the rest of the session. - */ - event[0] = lttng_kernel_event_create(&event_enabler->parent.parent, NULL, &event_pair); - if (IS_ERR(event[0])) { - lttng_event_enabler_destroy(&event_enabler->parent.parent); - ret = PTR_ERR(event[0]); - goto event_error; - } - - strcpy(event_pair.name, event_param->name); - strcat(event_pair.name, "_exit"); - event[1] = lttng_kernel_event_create(&event_enabler->parent.parent, NULL, &event_pair); - lttng_event_enabler_destroy(&event_enabler->parent.parent); - if (IS_ERR(event[1])) { - ret = PTR_ERR(event[1]); - goto event_error; - } - priv = event[0]; - break; - } - case LTTNG_KERNEL_ABI_FUNCTION: lttng_fallthrough; case LTTNG_KERNEL_ABI_NOOP: @@ -2596,11 +2517,11 @@ int lttng_abi_create_event_notifier(struct file *event_notifier_group_file, case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: fops = <tng_event_notifier_enabler_fops; break; - case LTTNG_KERNEL_ABI_KRETPROBE: - lttng_fallthrough; case LTTNG_KERNEL_ABI_UPROBE: fops = <tng_event_notifier_event_fops; break; @@ -2681,6 +2602,8 @@ int lttng_abi_create_event_notifier(struct file *event_notifier_group_file, } case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: { struct lttng_event_notifier_enabler *enabler; @@ -2713,48 +2636,6 @@ int lttng_abi_create_event_notifier(struct file *event_notifier_group_file, break; } - case LTTNG_KERNEL_ABI_KRETPROBE: - { - struct lttng_kernel_event_common *event[2]; - struct lttng_event_notifier_enabler *event_enabler; - struct lttng_kernel_event_pair event_pair; - - if (strlen(event_notifier_param->event.name) + strlen("_entry") >= LTTNG_KERNEL_ABI_SYM_NAME_LEN) - return -EINVAL; - - memset(&event_pair, 0, sizeof(event_pair)); - event_enabler = lttng_event_notifier_enabler_create(LTTNG_ENABLER_FORMAT_NAME, - event_notifier_param, event_notifier_group); - if (!event_enabler) { - ret = -ENOMEM; - goto event_notifier_error; - } - - strcpy(event_pair.name, event_notifier_param->event.name); - strcat(event_pair.name, "_entry"); - /* - * We tolerate no failure path after event creation. It - * will stay invariant for the rest of the session. - */ - event[0] = lttng_kernel_event_create(&event_enabler->parent, NULL, &event_pair); - if (IS_ERR(event[0])) { - lttng_event_enabler_destroy(&event_enabler->parent); - ret = PTR_ERR(event[0]); - goto event_notifier_error; - } - - strcpy(event_pair.name, event_notifier_param->event.name); - strcat(event_pair.name, "_exit"); - event[1] = lttng_kernel_event_create(&event_enabler->parent, NULL, &event_pair); - lttng_event_enabler_destroy(&event_enabler->parent); - if (IS_ERR(event[1])) { - ret = PTR_ERR(event[1]); - goto event_notifier_error; - } - priv = event[0]; - break; - } - case LTTNG_KERNEL_ABI_FUNCTION: lttng_fallthrough; case LTTNG_KERNEL_ABI_NOOP: diff --git a/src/lttng-events.c b/src/lttng-events.c index 49888e32..164ab196 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -690,6 +690,8 @@ int lttng_event_enable(struct lttng_kernel_event_common *event) case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: ret = -EINVAL; break; @@ -697,10 +699,6 @@ int lttng_event_enable(struct lttng_kernel_event_common *event) WRITE_ONCE(event->enabled, 1); break; - case LTTNG_KERNEL_ABI_KRETPROBE: - ret = lttng_kretprobes_event_enable_state(event, 1); - break; - case LTTNG_KERNEL_ABI_FUNCTION: lttng_fallthrough; case LTTNG_KERNEL_ABI_NOOP: @@ -749,6 +747,8 @@ int lttng_event_disable(struct lttng_kernel_event_common *event) case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: ret = -EINVAL; break; @@ -756,10 +756,6 @@ int lttng_event_disable(struct lttng_kernel_event_common *event) WRITE_ONCE(event->enabled, 0); break; - case LTTNG_KERNEL_ABI_KRETPROBE: - ret = lttng_kretprobes_event_enable_state(event, 0); - break; - case LTTNG_KERNEL_ABI_FUNCTION: lttng_fallthrough; case LTTNG_KERNEL_ABI_NOOP: @@ -1397,7 +1393,7 @@ struct lttng_kernel_event_common *_lttng_kernel_event_create(struct lttng_event_ const char *event_name; int ret; - if (event_pair == NULL || event_pair->refcount == 0) { + if (event_pair == NULL || event_pair->check_ids) { if (!lttng_kernel_event_id_available(event_enabler)) { ret = -EMFILE; goto full; @@ -1492,38 +1488,18 @@ struct lttng_kernel_event_common *_lttng_kernel_event_create(struct lttng_event_ case LTTNG_KERNEL_ABI_KRETPROBE: { - /* - * Needs to be explicitly enabled after creation, since - * we may want to apply filters. - */ + /* Event will be enabled by enabler sync. */ event->enabled = 0; - event->priv->registered = 1; - event->priv->desc = lttng_create_kretprobes_event_desc(event_name); - if (!event->priv->desc) { - ret = -ENOMEM; + event->priv->registered = 0; + ret = lttng_kretprobes_init_event(event_name, + event_pair->entryexit, + event, event_pair->krp); + if (ret) { + ret = -EINVAL; goto register_error; } - event_pair->event[event_pair->refcount++] = event; - - /* kretprobe defines 2 events. */ - if (event_pair->refcount == 2) { - /* - * Populate lttng_event structure before kretprobe registration. - */ - smp_wmb(); - ret = lttng_kretprobes_register(event_param->u.kretprobe.symbol_name, - event_param->u.kretprobe.offset, - event_param->u.kretprobe.addr, - event_pair->event[0], event_pair->event[1]); - if (ret) { - ret = -EINVAL; - goto register_error; - } - } ret = try_module_get(event->priv->desc->owner); WARN_ON_ONCE(!ret); - ret = lttng_append_event_to_channel_map(event_enabler, event, event_name); - WARN_ON_ONCE(ret); break; } @@ -1699,16 +1675,7 @@ void register_event(struct lttng_kernel_event_common *event) break; case LTTNG_KERNEL_ABI_KRETPROBE: - switch (event->type) { - case LTTNG_KERNEL_EVENT_TYPE_RECORDER: - lttng_fallthrough; - case LTTNG_KERNEL_EVENT_TYPE_COUNTER: - ret = 0; - break; - case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: - WARN_ON_ONCE(1); - break; - } + ret = lttng_kretprobes_register_event(event); break; case LTTNG_KERNEL_ABI_FUNCTION: @@ -1746,17 +1713,8 @@ void unregister_event(struct lttng_kernel_event_common *event) break; case LTTNG_KERNEL_ABI_KRETPROBE: - switch (event->type) { - case LTTNG_KERNEL_EVENT_TYPE_RECORDER: - lttng_fallthrough; - case LTTNG_KERNEL_EVENT_TYPE_COUNTER: - lttng_fallthrough; - case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: - lttng_kretprobes_unregister(event); - ret = 0; - break; - break; - } + lttng_kretprobes_unregister_event(event); + ret = 0; break; case LTTNG_KERNEL_ABI_SYSCALL: @@ -1846,7 +1804,7 @@ void _lttng_event_destroy(struct lttng_kernel_event_common *event) case LTTNG_KERNEL_ABI_KRETPROBE: module_put(event->priv->desc->owner); - lttng_kretprobes_destroy_private(event); + lttng_kretprobes_destroy_event_private(event); break; case LTTNG_KERNEL_ABI_SYSCALL: @@ -2240,6 +2198,53 @@ int lttng_desc_match_enabler_check(const struct lttng_kernel_event_desc *desc, } break; + case LTTNG_KERNEL_ABI_KRETPROBE: + { + char base_name[LTTNG_KERNEL_ABI_SYM_NAME_LEN]; + size_t base_name_len; /* includes \0 */ + char *last_separator, *entryexit; + + desc_name = desc->event_name; + last_separator = strrchr(desc_name, '_'); + base_name_len = last_separator - desc_name + 1; + memcpy(base_name, desc_name, base_name_len); + base_name[base_name_len - 1] = '\0'; /* Replace '_' by '\0' */ + entryexit = last_separator + 1; + + if (!strcmp(entryexit, "entry")) { + entry = true; + } else if (!strcmp(entryexit, "exit")) { + /* Nothing to do. */ + } else { + WARN_ON_ONCE(1); + return -EINVAL; + } + + switch (enabler->event_param.u.kretprobe.entryexit) { + case LTTNG_KERNEL_ABI_KRETPROBE_ENTRYEXIT: + break; + case LTTNG_KERNEL_ABI_KRETPROBE_ENTRY: + if (!entry) + return 0; + break; + case LTTNG_KERNEL_ABI_KRETPROBE_EXIT: + if (entry) + return 0; + break; + default: + return -EINVAL; + } + + switch (enabler->format_type) { + case LTTNG_ENABLER_FORMAT_STAR_GLOB: + return -EINVAL; + case LTTNG_ENABLER_FORMAT_NAME: + return lttng_match_enabler_name(base_name, enabler_name); + default: + return -EINVAL; + } + break; + } default: WARN_ON_ONCE(1); @@ -2383,6 +2388,53 @@ void lttng_event_enabler_create_kprobe_event_if_missing(struct lttng_event_enabl } } +/* Try to create the event associated with this kretprobe enabler. */ +static +void lttng_event_enabler_create_kretprobe_event_if_missing(struct lttng_event_enabler_common *event_enabler) +{ + struct lttng_kernel_abi_event *event_param = &event_enabler->event_param; + struct lttng_kernel_event_pair event_pair; + struct lttng_kernel_event_common *event; + + if (strlen(event_param->name) + strlen("_entry") >= LTTNG_KERNEL_ABI_SYM_NAME_LEN) { + WARN_ON_ONCE(1); + return; + } + + memset(&event_pair, 0, sizeof(event_pair)); + event_pair.krp = lttng_kretprobes_create_krp(event_param->u.kretprobe.symbol_name, + event_param->u.kretprobe.offset, event_param->u.kretprobe.addr); + if (!event_pair.krp) { + WARN_ON_ONCE(1); + return; + } + strcpy(event_pair.name, event_enabler->event_param.name); + strcat(event_pair.name, "_entry"); + event_pair.check_ids = true; + event_pair.entryexit = LTTNG_KRETPROBE_ENTRY; + event = _lttng_kernel_event_create(event_enabler, NULL, &event_pair); + if (IS_ERR(event)) { + if (PTR_ERR(event) != -EEXIST) { + printk(KERN_INFO "LTTng: Unable to create kretprobe event %s\n", + event_enabler->event_param.name); + } + } + + strcpy(event_pair.name, event_enabler->event_param.name); + strcat(event_pair.name, "_exit"); + event_pair.check_ids = false; + event_pair.entryexit = LTTNG_KRETPROBE_EXIT; + event = _lttng_kernel_event_create(event_enabler, NULL, &event_pair); + if (IS_ERR(event)) { + if (PTR_ERR(event) != -EEXIST) { + printk(KERN_INFO "LTTng: Unable to create kretprobe event %s\n", + event_enabler->event_param.name); + } + } + + lttng_kretprobes_put_krp(event_pair.krp); +} + /* * Create event if it is missing and present in the list of tracepoint probes. * Should be called with sessions mutex held. @@ -2406,6 +2458,10 @@ void lttng_event_enabler_create_events_if_missing(struct lttng_event_enabler_com lttng_event_enabler_create_kprobe_event_if_missing(event_enabler); break; + case LTTNG_KERNEL_ABI_KRETPROBE: + lttng_event_enabler_create_kretprobe_event_if_missing(event_enabler); + break; + default: WARN_ON_ONCE(1); break; @@ -2946,6 +3002,8 @@ bool lttng_get_event_enabled_state(struct lttng_kernel_event_common *event) case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: /* Enable events */ list_for_each_entry(enabler_ref, &event->priv->enablers_ref_head, node) { if (enabler_ref->ref->enabled) { @@ -2991,6 +3049,8 @@ bool lttng_event_is_lazy_sync(struct lttng_kernel_event_common *event) case LTTNG_KERNEL_ABI_SYSCALL: lttng_fallthrough; case LTTNG_KERNEL_ABI_KPROBE: + lttng_fallthrough; + case LTTNG_KERNEL_ABI_KRETPROBE: return true; default: diff --git a/src/lttng-kretprobes.c b/src/lttng-kretprobes.c index 54ef1df4..9cb04ad3 100644 --- a/src/lttng-kretprobes.c +++ b/src/lttng-kretprobes.c @@ -28,8 +28,14 @@ enum lttng_kretprobe_type { struct lttng_krp { struct kretprobe krp; - struct lttng_kernel_event_common *event[2]; /* ENTRY and EXIT */ - struct kref kref_register; + char *symbol_name; + uint64_t addr; + uint64_t offset; + kretprobe_handler_t entry_handler; + kretprobe_handler_t exit_handler; + struct lttng_kernel_event_common *event[NR_LTTNG_KRETPROBE_ENTRYEXIT]; /* entry/exit events */ + + int refcount_register; /* register kretprobe when > 0 */ struct kref kref_alloc; }; @@ -157,189 +163,169 @@ static const struct lttng_kernel_tracepoint_class tp_class = { .fields = event_fields, }; -/* - * Create event description - */ -struct lttng_kernel_event_desc *lttng_create_kretprobes_event_desc(const char *name) -{ - struct lttng_kernel_event_desc *desc; - char *alloc_name; - size_t name_len; - - desc = kzalloc(sizeof(*desc), GFP_KERNEL); - if (!desc) - return NULL; - name_len = strlen(name); - alloc_name = kmalloc(name_len + 1, GFP_KERNEL); - if (!alloc_name) - goto error_str; - strcpy(alloc_name, name); - desc->event_name = alloc_name; - desc->tp_class = &tp_class; - desc->owner = THIS_MODULE; - return desc; - -error_str: - kfree(desc); - return NULL; -} - static -int _lttng_kretprobes_register(const char *symbol_name, - uint64_t offset, - uint64_t addr, - struct lttng_krp *lttng_krp, - kretprobe_handler_t entry_handler, - kretprobe_handler_t exit_handler, - struct lttng_kernel_event_common *event_entry, - struct lttng_kernel_event_common *event_exit) +int lttng_krp_init(struct lttng_krp *lttng_krp, + const char *symbol_name, uint64_t offset, uint64_t addr, + kretprobe_handler_t entry_handler, + kretprobe_handler_t exit_handler) { - int ret; - - /* Kprobes expects a NULL symbol name if unused */ + /* Kretprobes expects a NULL symbol name if unused */ if (symbol_name[0] == '\0') symbol_name = NULL; - lttng_krp->krp.entry_handler = entry_handler; - lttng_krp->krp.handler = exit_handler; + + lttng_krp->entry_handler = entry_handler; + lttng_krp->exit_handler = exit_handler; + if (symbol_name) { - char *alloc_symbol; - - alloc_symbol = kstrdup(symbol_name, GFP_KERNEL); - if (!alloc_symbol) { - ret = -ENOMEM; - goto name_error; - } - lttng_krp->krp.kp.symbol_name = alloc_symbol; - if (event_entry) - event_entry->priv->u.kretprobe.symbol_name = alloc_symbol; - if (event_exit) - event_exit->priv->u.kretprobe.symbol_name = alloc_symbol; + lttng_krp->symbol_name = + kzalloc(LTTNG_KERNEL_ABI_SYM_NAME_LEN * sizeof(char), + GFP_KERNEL); + if (!lttng_krp->symbol_name) + return -ENOMEM; + memcpy(lttng_krp->symbol_name, symbol_name, + LTTNG_KERNEL_ABI_SYM_NAME_LEN * sizeof(char)); } - lttng_krp->krp.kp.offset = offset; - lttng_krp->krp.kp.addr = (void *) (unsigned long) addr; - /* Allow probe handler to find event structures */ - lttng_krp->event[EVENT_ENTRY] = event_entry; - lttng_krp->event[EVENT_EXIT] = event_exit; - if (event_entry) - event_entry->priv->u.kretprobe.lttng_krp = lttng_krp; - if (event_exit) - event_exit->priv->u.kretprobe.lttng_krp = lttng_krp; + lttng_krp->offset = offset; + lttng_krp->addr = addr; + return 0; +} - /* - * Both events must be unregistered before the kretprobe is - * unregistered. Same for memory allocation. - */ +struct lttng_krp *lttng_kretprobes_create_krp(const char *symbol_name, + uint64_t offset, uint64_t addr) +{ + struct lttng_krp *lttng_krp; + int ret; + + lttng_krp = kzalloc(sizeof(*lttng_krp), GFP_KERNEL); + if (!lttng_krp) + return NULL; kref_init(<tng_krp->kref_alloc); - kref_get(<tng_krp->kref_alloc); /* inc refcount to 2, no overflow. */ - kref_init(<tng_krp->kref_register); - kref_get(<tng_krp->kref_register); /* inc refcount to 2, no overflow. */ + ret = lttng_krp_init(lttng_krp, symbol_name, offset, addr, + lttng_kretprobes_handler_entry, + lttng_kretprobes_handler_exit); + if (ret) + goto error_init; + return lttng_krp; - /* - * Ensure the memory we just allocated don't trigger page faults. - * Well.. kprobes itself puts the page fault handler on the blacklist, - * but we can never be too careful. - */ - wrapper_vmalloc_sync_mappings(); +error_init: + kfree(lttng_krp); + return NULL; +} - ret = register_kretprobe(<tng_krp->krp); - if (ret) - goto register_error; - return 0; +static +void _lttng_kretprobes_release(struct kref *kref) +{ + struct lttng_krp *lttng_krp = container_of(kref, struct lttng_krp, kref_alloc); -register_error: kfree(lttng_krp->krp.kp.symbol_name); -name_error: - return ret; + kfree(lttng_krp); } -int lttng_kretprobes_register(const char *symbol_name, - uint64_t offset, - uint64_t addr, - struct lttng_kernel_event_common *event_entry, - struct lttng_kernel_event_common *event_exit) +void lttng_kretprobes_put_krp(struct lttng_krp *krp) { - int ret = -ENOMEM; - struct lttng_krp *lttng_krp; + kref_put(&krp->kref_alloc, _lttng_kretprobes_release); +} - lttng_krp = kzalloc(sizeof(*lttng_krp), GFP_KERNEL); - if (!lttng_krp) - goto krp_error; - ret = _lttng_kretprobes_register(symbol_name, offset, addr, lttng_krp, - lttng_kretprobes_handler_entry, lttng_kretprobes_handler_exit, - event_entry, event_exit); - if (ret) - goto register_error; +/* + * Initialize event + */ +int lttng_kretprobes_init_event(const char *name, + enum lttng_kretprobe_entryexit entryexit, + struct lttng_kernel_event_common *event, + struct lttng_krp *krp) +{ + struct lttng_kernel_event_desc *desc; + int ret; + + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + desc->tp_class = &tp_class; + desc->event_name = kstrdup(name, GFP_KERNEL); + if (!desc->event_name) { + ret = -ENOMEM; + goto error_str; + } + desc->owner = THIS_MODULE; + event->priv->desc = desc; + event->priv->u.kretprobe.lttng_krp = krp; + event->priv->u.kretprobe.entryexit = entryexit; + kref_get(&krp->kref_alloc); + krp->event[entryexit] = event; return 0; -register_error: - kfree(lttng_krp); -krp_error: +error_str: + kfree(desc); return ret; } static -void _lttng_kretprobes_unregister_release(struct kref *kref) +int _lttng_kretprobes_register(struct lttng_krp *lttng_krp) { - struct lttng_krp *lttng_krp = - container_of(kref, struct lttng_krp, kref_register); - unregister_kretprobe(<tng_krp->krp); + if (lttng_krp->refcount_register++ != 0) + return 0; /* Already registered */ + + /* + * Ensure the memory we just allocated don't notify page faults. + * Well.. kprobes itself puts the page fault handler on the blacklist, + * but we can never be too careful. + */ + wrapper_vmalloc_sync_mappings(); + + /* + * Populate struct kprobe on each registration because kprobe internally + * does destructive changes to its state (e.g. addr=NULL). + */ + memset(<tng_krp->krp, 0, sizeof(lttng_krp->krp)); + lttng_krp->krp.kp.symbol_name = lttng_krp->symbol_name; + lttng_krp->krp.kp.addr = (void *)(unsigned long)lttng_krp->addr; + lttng_krp->krp.kp.offset = lttng_krp->offset; + lttng_krp->krp.entry_handler = lttng_krp->entry_handler; + lttng_krp->krp.handler = lttng_krp->exit_handler; + return register_kretprobe(<tng_krp->krp); } -void lttng_kretprobes_unregister(struct lttng_kernel_event_common *event) +int lttng_kretprobes_register_event(struct lttng_kernel_event_common *event) { - kref_put(&event->priv->u.kretprobe.lttng_krp->kref_register, - _lttng_kretprobes_unregister_release); + return _lttng_kretprobes_register(event->priv->u.kretprobe.lttng_krp); } -static -void _lttng_kretprobes_release(struct kref *kref) +void lttng_kretprobes_unregister_event(struct lttng_kernel_event_common *event) { - struct lttng_krp *lttng_krp = - container_of(kref, struct lttng_krp, kref_alloc); - kfree(lttng_krp->krp.kp.symbol_name); + struct lttng_krp *lttng_krp = event->priv->u.kretprobe.lttng_krp; + + WARN_ON_ONCE(!lttng_krp->refcount_register); + if (--lttng_krp->refcount_register != 0) + return; /* Already unregistered */ + unregister_kretprobe(<tng_krp->krp); } int lttng_kretprobes_match_check(const char *symbol_name, uint64_t offset, uint64_t addr) { struct lttng_krp lttng_krp; - int ret; + int ret = 0; memset(<tng_krp, 0, sizeof(lttng_krp)); - ret = _lttng_kretprobes_register(symbol_name, offset, addr, <tng_krp, NULL, NULL, - NULL, NULL); + ret = lttng_krp_init(<tng_krp, symbol_name, offset, addr, NULL, NULL); if (ret) - return -ENOENT; + return ret; + ret = _lttng_kretprobes_register(<tng_krp); + if (ret) { + ret = -ENOENT; + goto end; + } unregister_kretprobe(<tng_krp.krp); - kfree(lttng_krp.krp.kp.symbol_name); - return 0; +end: + kfree(lttng_krp.symbol_name); + return ret; } -void lttng_kretprobes_destroy_private(struct lttng_kernel_event_common *event) +void lttng_kretprobes_destroy_event_private(struct lttng_kernel_event_common *event) { kfree(event->priv->desc->event_name); kfree(event->priv->desc); - kref_put(&event->priv->u.kretprobe.lttng_krp->kref_alloc, - _lttng_kretprobes_release); -} - -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, - int enable) -{ - struct lttng_kernel_event_common *event_exit; - struct lttng_krp *lttng_krp; - - if (event->priv->instrumentation != LTTNG_KERNEL_ABI_KRETPROBE) { - return -EINVAL; - } - if (event->enabled == enable) { - return -EBUSY; - } - lttng_krp = event->priv->u.kretprobe.lttng_krp; - event_exit = lttng_krp->event[EVENT_EXIT]; - WRITE_ONCE(event->enabled, enable); - WRITE_ONCE(event_exit->enabled, enable); - return 0; + lttng_kretprobes_put_krp(event->priv->u.kretprobe.lttng_krp); } MODULE_LICENSE("GPL and additional rights"); -- 2.34.1