From e2d5dbc7d09c7aa4f7c391fcdd4dfc95ba1ed326 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 22 Apr 2021 13:52:43 -0400 Subject: [PATCH] Refactoring: combine probe callbacks Combine probe callbacks for event recorder and notifier. Also combine the code for args vs noargs probes, shrinking the number of probe implementations from 4 to 1. Signed-off-by: Mathieu Desnoyers Change-Id: Ic7c0494ffd171caca33cfd13a8a40771691d89b5 --- include/lttng/events.h | 31 +- include/lttng/tracepoint-event-impl.h | 540 +++++++------------------- src/lttng-abi.c | 10 +- src/lttng-events.c | 60 ++- src/lttng-syscalls.c | 490 +++-------------------- src/probes/lttng-kprobes.c | 97 +++-- src/probes/lttng-kretprobes.c | 91 +++-- src/probes/lttng-uprobes.c | 114 +++--- 8 files changed, 408 insertions(+), 1025 deletions(-) diff --git a/include/lttng/events.h b/include/lttng/events.h index ee16fc2f..811f78ce 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -328,8 +328,7 @@ struct lttng_perf_counter_field { }; struct lttng_probe_ctx { - struct lttng_kernel_event_recorder *event; - struct lttng_kernel_event_notifier *event_notifier; // Not sure if we will ever need it. + struct lttng_kernel_event_common *event; uint8_t interruptible; }; @@ -364,7 +363,6 @@ struct lttng_kernel_event_desc { const struct lttng_kernel_event_field **fields; /* event payload */ unsigned int nr_fields; struct module *owner; - void *event_notifier_callback; }; struct lttng_kernel_probe_desc { @@ -432,10 +430,7 @@ struct lttng_enabler_ref { }; struct lttng_uprobe_handler { - union { - struct lttng_kernel_event_recorder *event; - struct lttng_kernel_event_notifier *event_notifier; - } u; + struct lttng_kernel_event_common *event; loff_t offset; struct uprobe_consumer up_consumer; struct list_head node; @@ -1331,23 +1326,18 @@ void lttng_kprobes_destroy_event_notifier_private(struct lttng_kernel_event_noti } #endif -int lttng_event_add_callsite(struct lttng_kernel_event_recorder *event, - struct lttng_kernel_abi_event_callsite __user *callsite); - -int lttng_event_notifier_add_callsite(struct lttng_kernel_event_notifier *event_notifier, +int lttng_event_add_callsite(struct lttng_kernel_event_common *event, struct lttng_kernel_abi_event_callsite __user *callsite); #ifdef CONFIG_UPROBES int lttng_uprobes_register_event(const char *name, int fd, struct lttng_kernel_event_recorder *event); -int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_recorder *event, +int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_common *event, struct lttng_kernel_abi_event_callsite __user *callsite); void lttng_uprobes_unregister_event(struct lttng_kernel_event_recorder *event); void lttng_uprobes_destroy_event_private(struct lttng_kernel_event_recorder *event); int lttng_uprobes_register_event_notifier(const char *name, int fd, struct lttng_kernel_event_notifier *event_notifier); -int lttng_uprobes_event_notifier_add_callsite(struct lttng_kernel_event_notifier *event_notifier, - struct lttng_kernel_abi_event_callsite __user *callsite); void lttng_uprobes_unregister_event_notifier(struct lttng_kernel_event_notifier *event_notifier); void lttng_uprobes_destroy_event_notifier_private(struct lttng_kernel_event_notifier *event_notifier); #else @@ -1359,7 +1349,7 @@ int lttng_uprobes_register_event(const char *name, } static inline -int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_recorder *event, +int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_common *event, struct lttng_kernel_abi_event_callsite __user *callsite) { return -ENOSYS; @@ -1382,13 +1372,6 @@ int lttng_uprobes_register_event_notifier(const char *name, return -ENOSYS; } -static inline -int lttng_uprobes_event_notifier_add_callsite(struct lttng_kernel_event_notifier *event_notifier, - struct lttng_kernel_abi_event_callsite __user *callsite) -{ - return -ENOSYS; -} - static inline void lttng_uprobes_unregister_event_notifier(struct lttng_kernel_event_notifier *event_notifier) { @@ -1409,7 +1392,7 @@ int lttng_kretprobes_register(const char *name, struct lttng_kernel_event_recorder *event_exit); void lttng_kretprobes_unregister(struct lttng_kernel_event_recorder *event); void lttng_kretprobes_destroy_private(struct lttng_kernel_event_recorder *event); -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_recorder *event, +int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, int enable); #else static inline @@ -1434,7 +1417,7 @@ void lttng_kretprobes_destroy_private(struct lttng_kernel_event_recorder *event) } static inline -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_recorder *event, +int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, int enable) { return -ENOSYS; diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index c9674a35..f23c2677 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -173,41 +173,6 @@ void __event_template_proto___##_name(void); #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -/* - * Stage 1.2 of the trace event_notifier. - * - * Create dummy trace prototypes for each event class, and for each used - * template. This will allow checking whether the prototypes from the - * class and the instance using the class actually match. - */ - -#include /* Reset all macros within TRACE_EVENT */ - -#undef TP_PROTO -#define TP_PROTO(...) __VA_ARGS__ - -#undef TP_ARGS -#define TP_ARGS(...) __VA_ARGS__ - -#undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP -#define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \ -void __event_notifier_template_proto___##_template(_proto); - -#undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS -#define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \ -void __event_notifier_template_proto___##_template(void); - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -void __event_notifier_template_proto___##_name(_proto); - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -void __event_notifier_template_proto___##_name(void); - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - - /* * Stage 1.2 of tracepoint event generation * @@ -377,28 +342,6 @@ static void __event_probe__##_name(void *__data); #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -/* - * Stage 3.1 of the trace event_notifiers. - * - * Create event_notifier probe callback prototypes. - */ - -/* Reset all macros within TRACEPOINT_EVENT */ -#include - -#undef TP_PROTO -#define TP_PROTO(...) __VA_ARGS__ - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static void __event_notifier_probe__##_name(void *__data, _proto); - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static void __event_notifier_probe__##_name(void *__data); - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - /* * Stage 4 of the trace events. * @@ -1055,348 +998,160 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ * 2*sizeof(unsigned long) for all supported architectures. * Perform UNION (||) of filter runtime list. */ +#undef _LTTNG_TRACEPOINT_EVENT_CLASS_CODE +#define _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _data_proto, _locvar_args, _locvar, _code_pre, _fields, _code_post) \ +static void __event_probe__##_name(_data_proto) \ +{ \ + struct probe_local_vars { _locvar }; \ + struct lttng_kernel_event_common *__event = __data; \ + struct lttng_probe_ctx __lttng_probe_ctx = { \ + .event = __event, \ + .interruptible = !irqs_disabled(), \ + }; \ + union { \ + size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ + char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ + } __stackvar; \ + size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ + struct probe_local_vars __tp_locvar; \ + struct probe_local_vars *tp_locvar __attribute__((unused)) = \ + &__tp_locvar; \ + bool __interpreter_stack_prepared = false; \ + \ + switch (__event->type) { \ + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: \ + { \ + struct lttng_kernel_event_recorder *__event_recorder = \ + container_of(__event, struct lttng_kernel_event_recorder, parent); \ + struct lttng_channel *__chan = __event_recorder->chan; \ + struct lttng_session *__session = __chan->session; \ + struct lttng_id_tracker_rcu *__lf; \ + \ + if (!_TP_SESSION_CHECK(session, __session)) \ + return; \ + if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ + return; \ + if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ + return; \ + __lf = lttng_rcu_dereference(__session->pid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \ + return; \ + __lf = lttng_rcu_dereference(__session->vpid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \ + return; \ + __lf = lttng_rcu_dereference(__session->uid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ + lttng_current_uid()))) \ + return; \ + __lf = lttng_rcu_dereference(__session->vuid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ + lttng_current_vuid()))) \ + return; \ + __lf = lttng_rcu_dereference(__session->gid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ + lttng_current_gid()))) \ + return; \ + __lf = lttng_rcu_dereference(__session->vgid_tracker.p); \ + if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ + lttng_current_vgid()))) \ + return; \ + break; \ + } \ + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: \ + break; \ + default: \ + WARN_ON_ONCE(1); \ + } \ + if (unlikely(!READ_ONCE(__event->enabled))) \ + return; \ + __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ + __dynamic_len_idx = __orig_dynamic_len_offset; \ + _code_pre \ + if (unlikely(READ_ONCE(__event->eval_filter))) { \ + struct lttng_bytecode_runtime *bc_runtime; \ + bool __filter_record = false; \ + \ + __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ + _locvar_args); \ + __interpreter_stack_prepared = true; \ + lttng_list_for_each_entry_rcu(bc_runtime, &__event->priv->filter_bytecode_runtime_head, node) { \ + if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \ + __filter_record = true; \ + break; \ + } \ + } \ + if (likely(!__filter_record)) \ + goto __post; \ + } \ + switch (__event->type) { \ + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: \ + { \ + struct lttng_kernel_event_recorder *__event_recorder = \ + container_of(__event, struct lttng_kernel_event_recorder, parent); \ + struct lttng_channel *__chan = __event_recorder->chan; \ + struct lib_ring_buffer_ctx __ctx; \ + ssize_t __event_len; \ + size_t __event_align; \ + int __ret; \ + \ + __event_len = __event_get_size__##_name(_locvar_args); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ + __event_align = __event_get_align__##_name(_locvar_args); \ + lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ + __event_align, -1); \ + __ret = __chan->ops->event_reserve(&__ctx, __event_recorder->priv->id); \ + if (__ret < 0) \ + goto __post; \ + _fields \ + __chan->ops->event_commit(&__ctx); \ + break; \ + } \ + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: \ + { \ + struct lttng_kernel_event_notifier *__event_notifier = \ + container_of(__event, struct lttng_kernel_event_notifier, parent); \ + struct lttng_kernel_notification_ctx __notif_ctx; \ + \ + __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ + if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ + __event_prepare_interpreter_stack__##_name( \ + __stackvar.__interpreter_stack_data, \ + _locvar_args); \ + \ + __event_notifier->notification_send(__event_notifier, \ + &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data, \ + &__notif_ctx); \ + break; \ + } \ + default: \ + WARN_ON_ONCE(1); \ + } \ +__post: \ + _code_post \ + barrier(); /* use before un-reserve. */ \ + this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \ + return; \ +} + #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data, _proto) \ -{ \ - struct probe_local_vars { _locvar }; \ - struct lttng_kernel_event_recorder *__event_recorder = __data; \ - struct lttng_probe_ctx __lttng_probe_ctx = { \ - .event = __event_recorder, \ - .event_notifier = NULL, \ - .interruptible = !irqs_disabled(), \ - }; \ - struct lttng_channel *__chan = __event_recorder->chan; \ - struct lttng_session *__session = __chan->session; \ - struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ - size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ - union { \ - size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ - } __stackvar; \ - int __ret; \ - struct probe_local_vars __tp_locvar; \ - struct probe_local_vars *tp_locvar __attribute__((unused)) = \ - &__tp_locvar; \ - struct lttng_id_tracker_rcu *__lf; \ - \ - if (!_TP_SESSION_CHECK(session, __session)) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__event_recorder->parent.enabled))) \ - return; \ - __lf = lttng_rcu_dereference(__session->pid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vpid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \ - return; \ - __lf = lttng_rcu_dereference(__session->uid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_uid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vuid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_vuid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->gid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_gid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vgid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_vgid()))) \ - return; \ - __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ - __dynamic_len_idx = __orig_dynamic_len_offset; \ - _code_pre \ - if (unlikely(!list_empty(&__event_recorder->priv->parent.filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - int __filter_record = __event_recorder->priv->parent.has_enablers_without_filter_bytecode; \ - \ - __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \ - tp_locvar, _args); \ - lttng_list_for_each_entry_rcu(bc_runtime, &__event_recorder->priv->parent.filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \ - __filter_record = 1; \ - break; \ - } \ - } \ - if (likely(!__filter_record)) \ - goto __post; \ - } \ - __event_len = __event_get_size__##_name(tp_locvar, _args); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ - } \ - __event_align = __event_get_align__##_name(tp_locvar, _args); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event_recorder->priv->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ -__post: \ - _code_post \ - barrier(); /* use before un-reserve. */ \ - this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \ - return; \ -} + _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(void *__data, _proto), PARAMS(tp_locvar, _args), \ + PARAMS(_locvar), PARAMS(_code_pre), PARAMS(_fields), PARAMS(_code_post)) #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data) \ -{ \ - struct probe_local_vars { _locvar }; \ - struct lttng_kernel_event_recorder *__event_recorder = __data; \ - struct lttng_probe_ctx __lttng_probe_ctx = { \ - .event = __event_recorder, \ - .event_notifier = NULL, \ - .interruptible = !irqs_disabled(), \ - }; \ - struct lttng_channel *__chan = __event_recorder->chan; \ - struct lttng_session *__session = __chan->session; \ - struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ - size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ - union { \ - size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ - } __stackvar; \ - int __ret; \ - struct probe_local_vars __tp_locvar; \ - struct probe_local_vars *tp_locvar __attribute__((unused)) = \ - &__tp_locvar; \ - struct lttng_id_tracker_rcu *__lf; \ - \ - if (!_TP_SESSION_CHECK(session, __session)) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ - return; \ - if (unlikely(!LTTNG_READ_ONCE(__event_recorder->parent.enabled))) \ - return; \ - __lf = lttng_rcu_dereference(__session->pid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vpid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \ - return; \ - __lf = lttng_rcu_dereference(__session->uid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_uid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vuid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_vuid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->gid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_gid()))) \ - return; \ - __lf = lttng_rcu_dereference(__session->vgid_tracker.p); \ - if (__lf && likely(!lttng_id_tracker_lookup(__lf, \ - lttng_current_vgid()))) \ - return; \ - __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ - __dynamic_len_idx = __orig_dynamic_len_offset; \ - _code_pre \ - if (unlikely(!list_empty(&__event_recorder->priv->parent.filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - int __filter_record = __event_recorder->priv->parent.has_enablers_without_filter_bytecode; \ - \ - __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \ - tp_locvar); \ - lttng_list_for_each_entry_rcu(bc_runtime, &__event_recorder->priv->parent.filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \ - __filter_record = 1; \ - break; \ - } \ - } \ - if (likely(!__filter_record)) \ - goto __post; \ - } \ - __event_len = __event_get_size__##_name(tp_locvar); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ - } \ - __event_align = __event_get_align__##_name(tp_locvar); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event_recorder->priv->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ -__post: \ - _code_post \ - barrier(); /* use before un-reserve. */ \ - this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \ - return; \ -} + _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(void *__data), PARAMS(tp_locvar), PARAMS(_locvar), \ + PARAMS(_code_pre), PARAMS(_fields), PARAMS(_code_post)) #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) #undef __get_dynamic_len -/* - * Stage 6.1 of tracepoint generation: generate event notifier probes - * - * Create the probe function. This function evaluates the filter bytecode and - * queue a notification to be sent to userspace. - */ - -#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ - -#undef TP_PROTO -#define TP_PROTO(...) __VA_ARGS__ - -#undef TP_ARGS -#define TP_ARGS(...) __VA_ARGS__ - -#undef TP_FIELDS -#define TP_FIELDS(...) __VA_ARGS__ - -#undef TP_locvar -#define TP_locvar(...) __VA_ARGS__ - -#undef TP_code_pre -#define TP_code_pre(...) __VA_ARGS__ - -#undef TP_code_post -#define TP_code_post(...) __VA_ARGS__ - -/* - * Using twice size for filter stack data to hold size and pointer for - * each field (worse case). For integers, max size required is 64-bit. - * Same for double-precision floats. Those fit within - * 2*sizeof(unsigned long) for all supported architectures. - * Perform UNION (||) of filter runtime list. - */ -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static void __event_notifier_probe__##_name(void *__data, _proto) \ -{ \ - struct probe_local_vars { _locvar }; \ - struct lttng_kernel_event_notifier *__event_notifier = __data; \ - struct lttng_probe_ctx __lttng_probe_ctx = { \ - .event = NULL, \ - .event_notifier = __event_notifier, \ - .interruptible = !irqs_disabled(), \ - }; \ - union { \ - size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ - } __stackvar; \ - struct probe_local_vars __tp_locvar; \ - struct probe_local_vars *tp_locvar __attribute__((unused)) = \ - &__tp_locvar; \ - struct lttng_kernel_notification_ctx __notif_ctx; \ - bool __interpreter_stack_prepared = false; \ - \ - if (unlikely(!READ_ONCE(__event_notifier->parent.enabled))) \ - return; \ - _code_pre \ - if (unlikely(!list_empty(&__event_notifier->priv->parent.filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - int __filter_record = __event_notifier->priv->parent.has_enablers_without_filter_bytecode; \ - \ - __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ - tp_locvar, _args); \ - __interpreter_stack_prepared = true; \ - lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->priv->parent.filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ - __filter_record = 1; \ - } \ - if (likely(!__filter_record)) \ - goto __post; \ - } \ - \ - __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ - if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ - __event_prepare_interpreter_stack__##_name( \ - __stackvar.__interpreter_stack_data, \ - tp_locvar, _args); \ - \ - __event_notifier->notification_send(__event_notifier, \ - &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data, \ - &__notif_ctx); \ - \ -__post: \ - _code_post \ - return; \ -} - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static void __event_notifier_probe__##_name(void *__data) \ -{ \ - struct probe_local_vars { _locvar }; \ - struct lttng_kernel_event_notifier *__event_notifier = __data; \ - struct lttng_probe_ctx __lttng_probe_ctx = { \ - .event = NULL, \ - .event_notifier = __event_notifier, \ - .interruptible = !irqs_disabled(), \ - }; \ - union { \ - size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ - } __stackvar; \ - struct probe_local_vars __tp_locvar; \ - struct probe_local_vars *tp_locvar __attribute__((unused)) = \ - &__tp_locvar; \ - struct lttng_kernel_notification_ctx __notif_ctx; \ - bool __interpreter_stack_prepared = false; \ - \ - if (unlikely(!READ_ONCE(__event_notifier->parent.enabled))) \ - return; \ - _code_pre \ - if (unlikely(!list_empty(&__event_notifier->priv->parent.filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - int __filter_record = __event_notifier->priv->parent.has_enablers_without_filter_bytecode; \ - \ - __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ - tp_locvar); \ - __interpreter_stack_prepared = true; \ - lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->priv->parent.filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ - __filter_record = 1; \ - } \ - if (likely(!__filter_record)) \ - goto __post; \ - } \ - \ - __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ - if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ - __event_prepare_interpreter_stack__##_name( \ - __stackvar.__interpreter_stack_data, \ - tp_locvar); \ - \ - __event_notifier->notification_send(__event_notifier, \ - &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data, \ - &__notif_ctx); \ -__post: \ - _code_post \ - return; \ -} - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) /* * Stage 7 of the trace events. * @@ -1411,20 +1166,15 @@ __post: \ #define TP_PROBE_CB(_template) &__event_probe__##_template #endif -#ifndef TP_EVENT_NOTIFIER_PROBE_CB -#define TP_EVENT_NOTIFIER_PROBE_CB(_template) &__event_notifier_probe__##_template -#endif - #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \ static const struct lttng_kernel_event_desc __event_desc___##_map = { \ .event_name = #_map, \ .event_kname = #_name, \ - .probe_callback = (void *) TP_PROBE_CB(_template), \ + .probe_callback = (void (*)(void)) TP_PROBE_CB(_template), \ .fields = __event_fields___##_template, \ .nr_fields = ARRAY_SIZE(__event_fields___##_template), \ .owner = THIS_MODULE, \ - .event_notifier_callback = (void *) TP_EVENT_NOTIFIER_PROBE_CB(_template), \ }; #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP diff --git a/src/lttng-abi.c b/src/lttng-abi.c index e06a98b4..d660ad47 100644 --- a/src/lttng-abi.c +++ b/src/lttng-abi.c @@ -1912,7 +1912,7 @@ long lttng_event_notifier_event_ioctl(struct file *file, unsigned int cmd, unsig case LTTNG_KERNEL_ABI_CAPTURE: return -EINVAL; case LTTNG_KERNEL_ABI_ADD_CALLSITE: - return lttng_event_notifier_add_callsite(event_notifier, + return lttng_event_add_callsite(&event_notifier->parent, (struct lttng_kernel_abi_event_callsite __user *) arg); default: return -ENOIOCTLCMD; @@ -2582,7 +2582,7 @@ static const struct file_operations lttng_metadata_fops = { static long lttng_event_recorder_event_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - struct lttng_kernel_event_recorder *event = file->private_data; + struct lttng_kernel_event_recorder *event_recorder = file->private_data; switch (cmd) { case LTTNG_KERNEL_ABI_OLD_CONTEXT: @@ -2597,14 +2597,14 @@ long lttng_event_recorder_event_ioctl(struct file *file, unsigned int cmd, unsig } case LTTNG_KERNEL_ABI_OLD_ENABLE: case LTTNG_KERNEL_ABI_ENABLE: - return lttng_event_enable(event); + return lttng_event_enable(event_recorder); case LTTNG_KERNEL_ABI_OLD_DISABLE: case LTTNG_KERNEL_ABI_DISABLE: - return lttng_event_disable(event); + return lttng_event_disable(event_recorder); case LTTNG_KERNEL_ABI_FILTER: return -EINVAL; case LTTNG_KERNEL_ABI_ADD_CALLSITE: - return lttng_event_add_callsite(event, + return lttng_event_add_callsite(&event_recorder->parent, (struct lttng_kernel_abi_event_callsite __user *) arg); default: return -ENOIOCTLCMD; diff --git a/src/lttng-events.c b/src/lttng-events.c index 0c5b49d9..c956db54 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -629,7 +629,7 @@ int lttng_event_enable(struct lttng_kernel_event_recorder *event_recorder) break; case LTTNG_KERNEL_ABI_KRETPROBE: - ret = lttng_kretprobes_event_enable_state(event_recorder, 1); + ret = lttng_kretprobes_event_enable_state(&event_recorder->parent, 1); break; case LTTNG_KERNEL_ABI_FUNCTION: /* Fall-through */ @@ -670,7 +670,7 @@ int lttng_event_disable(struct lttng_kernel_event_recorder *event_recorder) case LTTNG_KERNEL_ABI_KRETPROBE: - ret = lttng_kretprobes_event_enable_state(event_recorder, 0); + ret = lttng_kretprobes_event_enable_state(&event_recorder->parent, 0); break; case LTTNG_KERNEL_ABI_FUNCTION: /* Fall-through */ @@ -1497,7 +1497,7 @@ void register_event_notifier(struct lttng_kernel_event_notifier *event_notifier) switch (event_notifier->priv->parent.instrumentation) { case LTTNG_KERNEL_ABI_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_register(desc->event_kname, - desc->event_notifier_callback, + desc->probe_callback, event_notifier); break; @@ -1534,7 +1534,7 @@ int _lttng_event_notifier_unregister( switch (event_notifier->priv->parent.instrumentation) { case LTTNG_KERNEL_ABI_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_unregister(event_notifier->priv->parent.desc->event_kname, - event_notifier->priv->parent.desc->event_notifier_callback, + event_notifier->priv->parent.desc->probe_callback, event_notifier); break; @@ -2518,13 +2518,13 @@ error: return ret; } -int lttng_event_add_callsite(struct lttng_kernel_event_recorder *event_recorder, +int lttng_event_add_callsite(struct lttng_kernel_event_common *event, struct lttng_kernel_abi_event_callsite __user *callsite) { - switch (event_recorder->priv->parent.instrumentation) { + switch (event->priv->instrumentation) { case LTTNG_KERNEL_ABI_UPROBE: - return lttng_uprobes_event_add_callsite(event_recorder, callsite); + return lttng_uprobes_event_add_callsite(event, callsite); default: return -EINVAL; } @@ -2666,19 +2666,6 @@ end: return ret; } -int lttng_event_notifier_add_callsite(struct lttng_kernel_event_notifier *event_notifier, - struct lttng_kernel_abi_event_callsite __user *callsite) -{ - - switch (event_notifier->priv->parent.instrumentation) { - case LTTNG_KERNEL_ABI_UPROBE: - return lttng_uprobes_event_notifier_add_callsite(event_notifier, - callsite); - default: - return -EINVAL; - } -} - static void lttng_event_notifier_enabler_destroy( struct lttng_event_notifier_enabler *event_notifier_enabler) @@ -2715,7 +2702,8 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; struct lttng_enabler_ref *enabler_ref; struct lttng_bytecode_runtime *runtime; - int enabled = 0, has_enablers_without_bytecode = 0; + int enabled = 0, has_enablers_without_filter_bytecode = 0; + int nr_filters = 0; switch (event_recorder_priv->parent.instrumentation) { case LTTNG_KERNEL_ABI_TRACEPOINT: /* Fall-through */ @@ -2757,17 +2745,21 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) &event_recorder_priv->parent.enablers_ref_head, node) { if (enabler_ref->ref->enabled && list_empty(&enabler_ref->ref->filter_bytecode_head)) { - has_enablers_without_bytecode = 1; + has_enablers_without_filter_bytecode = 1; break; } } event_recorder_priv->parent.has_enablers_without_filter_bytecode = - has_enablers_without_bytecode; + has_enablers_without_filter_bytecode; /* Enable filters */ list_for_each_entry(runtime, - &event_recorder_priv->parent.filter_bytecode_runtime_head, node) + &event_recorder_priv->parent.filter_bytecode_runtime_head, node) { lttng_bytecode_filter_sync_state(runtime); + nr_filters++; + } + WRITE_ONCE(event_recorder_priv->parent.pub->eval_filter, + !(has_enablers_without_filter_bytecode || !nr_filters)); } } @@ -2804,7 +2796,8 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; struct lttng_enabler_ref *enabler_ref; struct lttng_bytecode_runtime *runtime; - int enabled = 0, has_enablers_without_bytecode = 0; + int enabled = 0, has_enablers_without_filter_bytecode = 0; + int nr_filters = 0, nr_captures = 0; switch (event_notifier_priv->parent.instrumentation) { case LTTNG_KERNEL_ABI_TRACEPOINT: /* Fall-through */ @@ -2842,24 +2835,29 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group &event_notifier_priv->parent.enablers_ref_head, node) { if (enabler_ref->ref->enabled && list_empty(&enabler_ref->ref->filter_bytecode_head)) { - has_enablers_without_bytecode = 1; + has_enablers_without_filter_bytecode = 1; break; } } event_notifier_priv->parent.has_enablers_without_filter_bytecode = - has_enablers_without_bytecode; + has_enablers_without_filter_bytecode; /* Enable filters */ list_for_each_entry(runtime, - &event_notifier_priv->parent.filter_bytecode_runtime_head, node) + &event_notifier_priv->parent.filter_bytecode_runtime_head, node) { lttng_bytecode_filter_sync_state(runtime); + nr_filters++; + } + WRITE_ONCE(event_notifier_priv->parent.pub->eval_filter, + !(has_enablers_without_filter_bytecode || !nr_filters)); /* Enable captures */ list_for_each_entry(runtime, - &event_notifier_priv->capture_bytecode_runtime_head, node) + &event_notifier_priv->capture_bytecode_runtime_head, node) { lttng_bytecode_capture_sync_state(runtime); - - WRITE_ONCE(event_notifier->eval_capture, !!event_notifier_priv->num_captures); + nr_captures++; + } + WRITE_ONCE(event_notifier->eval_capture, !!nr_captures); } } diff --git a/src/lttng-syscalls.c b/src/lttng-syscalls.c index d4914037..dfb00cbb 100644 --- a/src/lttng-syscalls.c +++ b/src/lttng-syscalls.c @@ -293,7 +293,6 @@ typedef __kernel_old_time_t time_t; struct trace_syscall_entry { void *event_func; - void *event_notifier_func; const struct lttng_kernel_event_desc *desc; const struct lttng_kernel_event_field **fields; unsigned int nrargs; @@ -310,7 +309,6 @@ struct trace_syscall_entry { #define TRACE_SYSCALL_TABLE(_template, _name, _nr, _nrargs) \ [ _nr ] = { \ .event_func = __event_probe__syscall_entry_##_template, \ - .event_notifier_func = __event_notifier_probe__syscall_entry_##_template, \ .nrargs = (_nrargs), \ .fields = __event_fields___syscall_entry_##_template, \ .desc = &__event_desc___syscall_entry_##_name, \ @@ -326,7 +324,6 @@ static const struct trace_syscall_entry sc_table[] = { #define TRACE_SYSCALL_TABLE(_template, _name, _nr, _nrargs) \ [ _nr ] = { \ .event_func = __event_probe__compat_syscall_entry_##_template, \ - .event_notifier_func = __event_notifier_probe__compat_syscall_entry_##_template, \ .nrargs = (_nrargs), \ .fields = __event_fields___compat_syscall_entry_##_template, \ .desc = &__event_desc___compat_syscall_entry_##_name, \ @@ -349,7 +346,6 @@ const struct trace_syscall_entry compat_sc_table[] = { #define TRACE_SYSCALL_TABLE(_template, _name, _nr, _nrargs) \ [ _nr ] = { \ .event_func = __event_probe__syscall_exit_##_template, \ - .event_notifier_func = __event_notifier_probe__syscall_exit_##_template, \ .nrargs = (_nrargs), \ .fields = __event_fields___syscall_exit_##_template, \ .desc = &__event_desc___syscall_exit_##_name, \ @@ -365,7 +361,6 @@ static const struct trace_syscall_entry sc_exit_table[] = { #define TRACE_SYSCALL_TABLE(_template, _name, _nr, _nrargs) \ [ _nr ] = { \ .event_func = __event_probe__compat_syscall_exit_##_template, \ - .event_notifier_func = __event_notifier_probe__compat_syscall_exit_##_template, \ .nrargs = (_nrargs), \ .fields = __event_fields___compat_syscall_exit_##_template, \ .desc = &__event_desc___compat_syscall_exit_##_name, \ @@ -396,59 +391,15 @@ static void syscall_entry_event_unknown(struct hlist_head *unknown_action_list_h lttng_syscall_get_arguments(current, regs, args); lttng_hlist_for_each_entry_rcu(event_priv, unknown_action_list_head, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - if (unlikely(in_compat_syscall())) - __event_probe__compat_syscall_entry_unknown(event_recorder, id, args); - else - __event_probe__syscall_entry_unknown(event_recorder, id, args); - } -} - -static void syscall_entry_event_notifier_unknown( - struct hlist_head *unknown_dispatch_list_head, - struct pt_regs *regs, long id) -{ - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - struct lttng_kernel_event_common_private *event_priv; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, unknown_dispatch_list_head, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - if (unlikely(in_compat_syscall())) - __event_notifier_probe__compat_syscall_entry_unknown(event_notifier, id, args); - else - __event_notifier_probe__syscall_entry_unknown(event_notifier, id, args); - } -} - -static void syscall_exit_event_notifier_unknown( - struct hlist_head *unknown_dispatch_list_head, - struct pt_regs *regs, long id, long ret) -{ - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - struct lttng_kernel_event_common_private *event_priv; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, unknown_dispatch_list_head, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - if (unlikely(in_compat_syscall())) - __event_notifier_probe__compat_syscall_exit_unknown(event_notifier, id, ret, args); + __event_probe__compat_syscall_entry_unknown(event_priv->pub, id, args); else - __event_notifier_probe__syscall_exit_unknown(event_notifier, id, ret, args); + __event_probe__syscall_entry_unknown(event_priv->pub, id, args); } } static __always_inline -void syscall_entry_call_func(struct hlist_head *action_list, +void syscall_entry_event_call_func(struct hlist_head *action_list, void *func, unsigned int nrargs, struct pt_regs *regs) { @@ -459,13 +410,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, { void (*fptr)(void *__data) = func; - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub); break; } case 1: @@ -474,13 +420,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0]); break; } case 2: @@ -491,13 +432,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0], args[1]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0], args[1]); break; } case 3: @@ -509,13 +445,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0], args[1], args[2]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0], args[1], args[2]); break; } case 4: @@ -528,13 +459,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0], args[1], args[2], args[3]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0], args[1], args[2], args[3]); break; } case 5: @@ -548,13 +474,8 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0], args[1], args[2], args[3], args[4]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0], args[1], args[2], args[3], args[4]); break; } case 6: @@ -569,149 +490,9 @@ void syscall_entry_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, args[0], args[1], args[2], + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, args[0], args[1], args[2], args[3], args[4], args[5]); - } - break; - } - default: - break; - } -} - -static __always_inline -void syscall_entry_event_notifier_call_func(struct hlist_head *dispatch_list, - void *func, unsigned int nrargs, struct pt_regs *regs) -{ - struct lttng_kernel_event_common_private *event_priv; - - switch (nrargs) { - case 0: - { - void (*fptr)(void *__data) = func; - - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier); - } - break; - } - case 1: - { - void (*fptr)(void *__data, unsigned long arg0) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0]); - } - break; - } - case 2: - { - void (*fptr)(void *__data, - unsigned long arg0, - unsigned long arg1) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0], args[1]); - } - break; - } - case 3: - { - void (*fptr)(void *__data, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0], args[1], args[2]); - } - break; - } - case 4: - { - void (*fptr)(void *__data, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0], args[1], args[2], args[3]); - } - break; - } - case 5: - { - void (*fptr)(void *__data, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3, - unsigned long arg4) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0], args[1], args[2], args[3], args[4]); - } - break; - } - case 6: - { - void (*fptr)(void *__data, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3, - unsigned long arg4, - unsigned long arg5) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, args[0], args[1], args[2], args[3], args[4], args[5]); - } break; } default: @@ -768,7 +549,7 @@ void syscall_entry_event_probe(void *__data, struct pt_regs *regs, long id) if (unlikely(hlist_empty(action_list))) return; - syscall_entry_call_func(action_list, entry->event_func, entry->nrargs, regs); + syscall_entry_event_call_func(action_list, entry->event_func, entry->nrargs, regs); } void syscall_entry_event_notifier_probe(void *__data, struct pt_regs *regs, @@ -806,14 +587,14 @@ void syscall_entry_event_notifier_probe(void *__data, struct pt_regs *regs, } /* Check if the syscall id is out of bound. */ if (unlikely(id < 0 || id >= table_len)) { - syscall_entry_event_notifier_unknown(unknown_dispatch_list, + syscall_entry_event_unknown(unknown_dispatch_list, regs, id); return; } entry = &table[id]; - if (!entry->event_notifier_func) { - syscall_entry_event_notifier_unknown(unknown_dispatch_list, + if (!entry->event_func) { + syscall_entry_event_unknown(unknown_dispatch_list, regs, id); return; } @@ -826,8 +607,8 @@ void syscall_entry_event_notifier_probe(void *__data, struct pt_regs *regs, if (unlikely(hlist_empty(dispatch_list))) return; - syscall_entry_event_notifier_call_func(dispatch_list, - entry->event_notifier_func, entry->nrargs, regs); + syscall_entry_event_call_func(dispatch_list, + entry->event_func, entry->nrargs, regs); } static void syscall_exit_event_unknown(struct hlist_head *unknown_action_list_head, @@ -838,20 +619,16 @@ static void syscall_exit_event_unknown(struct hlist_head *unknown_action_list_he lttng_syscall_get_arguments(current, regs, args); lttng_hlist_for_each_entry_rcu(event_priv, unknown_action_list_head, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - if (unlikely(in_compat_syscall())) - __event_probe__compat_syscall_exit_unknown(event_recorder, id, ret, + __event_probe__compat_syscall_exit_unknown(event_priv->pub, id, ret, args); else - __event_probe__syscall_exit_unknown(event_recorder, id, ret, args); + __event_probe__syscall_exit_unknown(event_priv->pub, id, ret, args); } } static __always_inline -void syscall_exit_call_func(struct hlist_head *action_list, +void syscall_exit_event_call_func(struct hlist_head *action_list, void *func, unsigned int nrargs, struct pt_regs *regs, long ret) { @@ -862,13 +639,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, { void (*fptr)(void *__data, long ret) = func; - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret); break; } case 1: @@ -879,13 +651,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0]); break; } case 2: @@ -897,13 +664,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0], args[1]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0], args[1]); break; } case 3: @@ -916,13 +678,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0], args[1], args[2]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0], args[1], args[2]); break; } case 4: @@ -936,13 +693,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0], args[1], args[2], args[3]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0], args[1], args[2], args[3]); break; } case 5: @@ -957,13 +709,8 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0], args[1], args[2], args[3], args[4]); - } + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0], args[1], args[2], args[3], args[4]); break; } case 6: @@ -979,14 +726,9 @@ void syscall_exit_call_func(struct hlist_head *action_list, unsigned long args[LTTNG_SYSCALL_NR_ARGS]; lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) { - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = event_recorder_priv->pub; - - fptr(event_recorder, ret, args[0], args[1], args[2], + lttng_hlist_for_each_entry_rcu(event_priv, action_list, u.syscall.node) + fptr(event_priv->pub, ret, args[0], args[1], args[2], args[3], args[4], args[5]); - } break; } default: @@ -1046,150 +788,10 @@ void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) if (unlikely(hlist_empty(action_list))) return; - syscall_exit_call_func(action_list, entry->event_func, entry->nrargs, + syscall_exit_event_call_func(action_list, entry->event_func, entry->nrargs, regs, ret); } -static __always_inline -void syscall_exit_event_notifier_call_func(struct hlist_head *dispatch_list, - void *func, unsigned int nrargs, struct pt_regs *regs, long ret) -{ - struct lttng_kernel_event_common_private *event_priv; - - switch (nrargs) { - case 0: - { - void (*fptr)(void *__data, long ret) = func; - - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret); - } - break; - } - case 1: - { - void (*fptr)(void *__data, long ret, unsigned long arg0) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0]); - } - break; - } - case 2: - { - void (*fptr)(void *__data, - long ret, - unsigned long arg0, - unsigned long arg1) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0], args[1]); - } - break; - } - case 3: - { - void (*fptr)(void *__data, - long ret, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0], args[1], args[2]); - } - break; - } - case 4: - { - void (*fptr)(void *__data, - long ret, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0], args[1], args[2], args[3]); - } - break; - } - case 5: - { - void (*fptr)(void *__data, - long ret, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3, - unsigned long arg4) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0], args[1], args[2], args[3], args[4]); - } - break; - } - case 6: - { - void (*fptr)(void *__data, - long ret, - unsigned long arg0, - unsigned long arg1, - unsigned long arg2, - unsigned long arg3, - unsigned long arg4, - unsigned long arg5) = func; - unsigned long args[LTTNG_SYSCALL_NR_ARGS]; - - lttng_syscall_get_arguments(current, regs, args); - lttng_hlist_for_each_entry_rcu(event_priv, dispatch_list, u.syscall.node) { - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = event_notifier_priv->pub; - - fptr(event_notifier, ret, args[0], args[1], args[2], args[3], args[4], args[5]); - } - break; - } - default: - break; - } -} - static void syscall_exit_event_notifier_probe(void *__data, struct pt_regs *regs, long ret) @@ -1229,14 +831,14 @@ void syscall_exit_event_notifier_probe(void *__data, struct pt_regs *regs, } /* Check if the syscall id is out of bound. */ if (unlikely(id < 0 || id >= table_len)) { - syscall_exit_event_notifier_unknown(unknown_dispatch_list, + syscall_exit_event_unknown(unknown_dispatch_list, regs, id, ret); return; } entry = &table[id]; - if (!entry->event_notifier_func) { - syscall_entry_event_notifier_unknown(unknown_dispatch_list, + if (!entry->event_func) { + syscall_entry_event_unknown(unknown_dispatch_list, regs, id); return; } @@ -1249,8 +851,8 @@ void syscall_exit_event_notifier_probe(void *__data, struct pt_regs *regs, if (unlikely(hlist_empty(dispatch_list))) return; - syscall_exit_event_notifier_call_func(dispatch_list, - entry->event_notifier_func, entry->nrargs, regs, ret); + syscall_exit_event_call_func(dispatch_list, + entry->event_func, entry->nrargs, regs, ret); } /* * noinline to diminish caller stack size. diff --git a/src/probes/lttng-kprobes.c b/src/probes/lttng-kprobes.c index ecb08fc8..d3a5cf8a 100644 --- a/src/probes/lttng-kprobes.c +++ b/src/probes/lttng-kprobes.c @@ -23,54 +23,69 @@ int lttng_kprobes_event_handler_pre(struct kprobe *p, struct pt_regs *regs) { struct lttng_kernel_event_common_private *event_priv = container_of(p, struct lttng_kernel_event_common_private, u.kprobe.kp); - struct lttng_kernel_event_recorder_private *event_recorder_priv = - container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); - struct lttng_kernel_event_recorder *event_recorder = - event_recorder_priv->pub; + struct lttng_kernel_event_common *event = event_priv->pub; struct lttng_probe_ctx lttng_probe_ctx = { - .event = event_recorder, + .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event_recorder->chan; - struct lib_ring_buffer_ctx ctx; - int ret; unsigned long data = (unsigned long) p->addr; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(event_recorder->parent.enabled))) - return 0; - - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), - lttng_alignof(data), -1); - ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); - if (ret < 0) - return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); - chan->ops->event_write(&ctx, &data, sizeof(data)); - chan->ops->event_commit(&ctx); - return 0; -} - -static -int lttng_kprobes_event_notifier_handler_pre(struct kprobe *p, struct pt_regs *regs) -{ - struct lttng_kernel_event_common_private *event_priv = - container_of(p, struct lttng_kernel_event_common_private, u.kprobe.kp); - struct lttng_kernel_event_notifier_private *event_notifier_priv = - container_of(event_priv, struct lttng_kernel_event_notifier_private, parent); - struct lttng_kernel_event_notifier *event_notifier = - event_notifier_priv->pub; - struct lttng_kernel_notification_ctx notif_ctx; + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder_private *event_recorder_priv = + container_of(event_priv, struct lttng_kernel_event_recorder_private, parent); + struct lttng_kernel_event_recorder *event_recorder = + event_recorder_priv->pub; + struct lttng_channel *chan = event_recorder->chan; + + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + return 0; + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + return 0; + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: + break; + default: + WARN_ON_ONCE(1); + } - if (unlikely(!READ_ONCE(event_notifier->parent.enabled))) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); - event_notifier->notification_send(event_notifier, NULL, NULL, ¬if_ctx); - + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder *event_recorder = + container_of(event, struct lttng_kernel_event_recorder, parent); + struct lttng_channel *chan = event_recorder->chan; + struct lib_ring_buffer_ctx ctx; + int ret; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), + lttng_alignof(data), -1); + ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); + chan->ops->event_write(&ctx, &data, sizeof(data)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: + { + struct lttng_kernel_event_notifier *event_notifier = + container_of(event, struct lttng_kernel_event_notifier, parent); + struct lttng_kernel_notification_ctx notif_ctx; + + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->notification_send(event_notifier, NULL, NULL, ¬if_ctx); + break; + } + default: + WARN_ON_ONCE(1); + } return 0; } @@ -244,7 +259,7 @@ int lttng_kprobes_register_event_notifier(const char *symbol_name, goto error; ret = _lttng_kprobes_register(symbol_name, offset, addr, - &event_notifier->priv->parent.u.kprobe, lttng_kprobes_event_notifier_handler_pre); + &event_notifier->priv->parent.u.kprobe, lttng_kprobes_event_handler_pre); if (ret) goto register_error; diff --git a/src/probes/lttng-kretprobes.c b/src/probes/lttng-kretprobes.c index 3084aedf..7bb9d5ca 100644 --- a/src/probes/lttng-kretprobes.c +++ b/src/probes/lttng-kretprobes.c @@ -26,7 +26,7 @@ enum lttng_kretprobe_type { struct lttng_krp { struct kretprobe krp; - struct lttng_kernel_event_recorder *event[2]; /* ENTRY and EXIT */ + struct lttng_kernel_event_common *event[2]; /* ENTRY and EXIT */ struct kref kref_register; struct kref kref_alloc; }; @@ -38,38 +38,63 @@ int _lttng_kretprobes_handler(struct kretprobe_instance *krpi, { struct lttng_krp *lttng_krp = container_of(lttng_get_kretprobe(krpi), struct lttng_krp, krp); - struct lttng_kernel_event_recorder *event_recorder = - lttng_krp->event[type]; + struct lttng_kernel_event_common *event = lttng_krp->event[type]; struct lttng_probe_ctx lttng_probe_ctx = { - .event = event_recorder, + .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event_recorder->chan; - struct lib_ring_buffer_ctx ctx; - int ret; struct { unsigned long ip; unsigned long parent_ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(event_recorder->parent.enabled))) - return 0; - - payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; - payload.parent_ip = (unsigned long) krpi->ret_addr; + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder *event_recorder = + container_of(event, struct lttng_kernel_event_recorder, parent); + struct lttng_channel *chan = event_recorder->chan; + + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + return 0; + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + return 0; + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: /* Fall-through. */ + default: + WARN_ON_ONCE(1); + } - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), - lttng_alignof(payload), -1); - ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); - if (ret < 0) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); + + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder *event_recorder = + container_of(event, struct lttng_kernel_event_recorder, parent); + struct lttng_channel *chan = event_recorder->chan; + struct lib_ring_buffer_ctx ctx; + int ret; + + payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; + payload.parent_ip = (unsigned long) krpi->ret_addr; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), + lttng_alignof(payload), -1); + ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: /* Fall-through. */ + default: + WARN_ON_ONCE(1); + } return 0; } @@ -212,8 +237,8 @@ int lttng_kretprobes_register(const char *name, lttng_krp->krp.kp.addr = (void *) (unsigned long) addr; /* Allow probe handler to find event structures */ - lttng_krp->event[EVENT_ENTRY] = event_recorder_entry; - lttng_krp->event[EVENT_EXIT] = event_recorder_exit; + lttng_krp->event[EVENT_ENTRY] = &event_recorder_entry->parent; + lttng_krp->event[EVENT_EXIT] = &event_recorder_exit->parent; event_recorder_entry->priv->parent.u.kretprobe.lttng_krp = lttng_krp; event_recorder_exit->priv->parent.u.kretprobe.lttng_krp = lttng_krp; @@ -294,22 +319,22 @@ void lttng_kretprobes_destroy_private(struct lttng_kernel_event_recorder *event_ } EXPORT_SYMBOL_GPL(lttng_kretprobes_destroy_private); -int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_recorder *event_recorder, +int lttng_kretprobes_event_enable_state(struct lttng_kernel_event_common *event, int enable) { - struct lttng_kernel_event_recorder *event_recorder_exit; + struct lttng_kernel_event_common *event_exit; struct lttng_krp *lttng_krp; - if (event_recorder->priv->parent.instrumentation != LTTNG_KERNEL_ABI_KRETPROBE) { + if (event->priv->instrumentation != LTTNG_KERNEL_ABI_KRETPROBE) { return -EINVAL; } - if (event_recorder->parent.enabled == enable) { + if (event->enabled == enable) { return -EBUSY; } - lttng_krp = event_recorder->priv->parent.u.kretprobe.lttng_krp; - event_recorder_exit = lttng_krp->event[EVENT_EXIT]; - WRITE_ONCE(event_recorder->parent.enabled, enable); - WRITE_ONCE(event_recorder_exit->parent.enabled, enable); + lttng_krp = event->priv->u.kretprobe.lttng_krp; + event_exit = lttng_krp->event[EVENT_EXIT]; + WRITE_ONCE(event->enabled, enable); + WRITE_ONCE(event_exit->enabled, enable); return 0; } EXPORT_SYMBOL_GPL(lttng_kretprobes_event_enable_state); diff --git a/src/probes/lttng-uprobes.c b/src/probes/lttng-uprobes.c index 58a5b35e..2507f434 100644 --- a/src/probes/lttng-uprobes.c +++ b/src/probes/lttng-uprobes.c @@ -28,55 +28,74 @@ int lttng_uprobes_event_handler_pre(struct uprobe_consumer *uc, struct pt_regs * { struct lttng_uprobe_handler *uprobe_handler = container_of(uc, struct lttng_uprobe_handler, up_consumer); - struct lttng_kernel_event_recorder *event_recorder = uprobe_handler->u.event; + struct lttng_kernel_event_common *event = uprobe_handler->event; struct lttng_probe_ctx lttng_probe_ctx = { - .event = event_recorder, + .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event_recorder->chan; - struct lib_ring_buffer_ctx ctx; - int ret; - struct { unsigned long ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) - return 0; - if (unlikely(!LTTNG_READ_ONCE(event_recorder->parent.enabled))) - return 0; - - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, - sizeof(payload), lttng_alignof(payload), -1); - - ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); - if (ret < 0) - return 0; - - /* Event payload. */ - payload.ip = (unsigned long)instruction_pointer(regs); - - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); - return 0; -} - -static -int lttng_uprobes_event_notifier_handler_pre(struct uprobe_consumer *uc, struct pt_regs *regs) -{ - struct lttng_uprobe_handler *uprobe_handler = - container_of(uc, struct lttng_uprobe_handler, up_consumer); - struct lttng_kernel_event_notifier *event_notifier = uprobe_handler->u.event_notifier; - struct lttng_kernel_notification_ctx notif_ctx; + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder *event_recorder = + container_of(event, struct lttng_kernel_event_recorder, parent); + struct lttng_channel *chan = event_recorder->chan; + + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + return 0; + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + return 0; + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: + break; + default: + WARN_ON_ONCE(1); + } - if (unlikely(!READ_ONCE(event_notifier->parent.enabled))) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); - event_notifier->notification_send(event_notifier, NULL, NULL, ¬if_ctx); + switch (event->type) { + case LTTNG_KERNEL_EVENT_TYPE_RECORDER: + { + struct lttng_kernel_event_recorder *event_recorder = + container_of(event, struct lttng_kernel_event_recorder, parent); + struct lttng_channel *chan = event_recorder->chan; + struct lib_ring_buffer_ctx ctx; + int ret; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, + sizeof(payload), lttng_alignof(payload), -1); + + ret = chan->ops->event_reserve(&ctx, event_recorder->priv->id); + if (ret < 0) + return 0; + + /* Event payload. */ + payload.ip = (unsigned long)instruction_pointer(regs); + + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER: + { + struct lttng_kernel_event_notifier *event_notifier = + container_of(event, struct lttng_kernel_event_notifier, parent); + struct lttng_kernel_notification_ctx notif_ctx; + + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->notification_send(event_notifier, NULL, NULL, ¬if_ctx); + break; + } + default: + WARN_ON_ONCE(1); + } return 0; } @@ -217,7 +236,7 @@ int lttng_uprobes_add_callsite(struct lttng_uprobe *uprobe, /* Ensure the memory we just allocated don't notify page faults. */ wrapper_vmalloc_sync_mappings(); - uprobe_handler->u.event = priv_data; + uprobe_handler->event = priv_data; uprobe_handler->up_consumer.handler = handler; ret = copy_from_user(&uprobe_handler->offset, &callsite->u.uprobe.offset, sizeof(uint64_t)); @@ -245,22 +264,14 @@ end: return ret; } -int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_recorder *event_recorder, +int lttng_uprobes_event_add_callsite(struct lttng_kernel_event_common *event, struct lttng_kernel_abi_event_callsite __user *callsite) { - return lttng_uprobes_add_callsite(&event_recorder->priv->parent.u.uprobe, callsite, - lttng_uprobes_event_handler_pre, event_recorder); + return lttng_uprobes_add_callsite(&event->priv->u.uprobe, callsite, + lttng_uprobes_event_handler_pre, event); } EXPORT_SYMBOL_GPL(lttng_uprobes_event_add_callsite); -int lttng_uprobes_event_notifier_add_callsite(struct lttng_kernel_event_notifier *event_notifier, - struct lttng_kernel_abi_event_callsite __user *callsite) -{ - return lttng_uprobes_add_callsite(&event_notifier->priv->parent.u.uprobe, callsite, - lttng_uprobes_event_notifier_handler_pre, event_notifier); -} -EXPORT_SYMBOL_GPL(lttng_uprobes_event_notifier_add_callsite); - static int lttng_uprobes_register(struct lttng_uprobe *uprobe, int fd) { @@ -339,7 +350,6 @@ void lttng_uprobes_unregister(struct inode *inode, struct list_head *head) list_del(&iter->node); kfree(iter); } - } void lttng_uprobes_unregister_event(struct lttng_kernel_event_recorder *event_recorder) -- 2.34.1