From: Mathieu Desnoyers Date: Fri, 23 Apr 2021 00:37:14 +0000 (-0400) Subject: Refactoring: bytecode interpreter X-Git-Tag: v2.13.0-rc1~3 X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=8a44545778a66f5c473b348ff0231145d59e1d21;p=lttng-modules.git Refactoring: bytecode interpreter Refactor bytecode interpreter to combine more code between filter and capture interpreters. Signed-off-by: Mathieu Desnoyers Change-Id: Ic9fe054ef4cbbef578e1e2c396bdadce221b9c6f --- diff --git a/include/lttng/events-internal.h b/include/lttng/events-internal.h index 30cf4adc..a583a2d3 100644 --- a/include/lttng/events-internal.h +++ b/include/lttng/events-internal.h @@ -20,7 +20,7 @@ struct lttng_kernel_event_common_private { uint64_t user_token; int has_enablers_without_filter_bytecode; - /* list of struct lttng_kernel_bytecode_runtime, sorted by seqnum */ + /* list of struct lttng_bytecode_runtime, sorted by seqnum */ struct list_head filter_bytecode_runtime_head; enum lttng_kernel_abi_instrumentation instrumentation; /* Selected by instrumentation */ @@ -64,6 +64,20 @@ struct lttng_kernel_event_notifier_private { }; +enum lttng_kernel_bytecode_interpreter_ret { + LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR = -1, + LTTNG_KERNEL_BYTECODE_INTERPRETER_OK = 0, +}; + +enum lttng_kernel_bytecode_filter_result { + LTTNG_KERNEL_BYTECODE_FILTER_ACCEPT = 0, + LTTNG_KERNEL_BYTECODE_FILTER_REJECT = 1, +}; + +struct lttng_kernel_bytecode_filter_ctx { + enum lttng_kernel_bytecode_filter_result result; +}; + static inline const struct lttng_kernel_type_integer *lttng_kernel_get_type_integer(const struct lttng_kernel_type_common *type) { @@ -138,4 +152,9 @@ static inline bool lttng_kernel_type_is_bytewise_integer(const struct lttng_kern return true; } +int lttng_kernel_interpret_event_filter(const struct lttng_kernel_event_common *event, + const char *interpreter_stack_data, + struct lttng_probe_ctx *probe_ctx, + void *event_filter_ctx); + #endif /* _LTTNG_EVENTS_INTERNAL_H */ diff --git a/include/lttng/events.h b/include/lttng/events.h index c396e790..2f48f6ef 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -376,13 +376,13 @@ struct lttng_kernel_probe_desc { struct lttng_krp; /* Kretprobe handling */ -enum lttng_bytecode_node_type { - LTTNG_BYTECODE_NODE_TYPE_FILTER, - LTTNG_BYTECODE_NODE_TYPE_CAPTURE, +enum lttng_kernel_bytecode_type { + LTTNG_KERNEL_BYTECODE_TYPE_FILTER, + LTTNG_KERNEL_BYTECODE_TYPE_CAPTURE, }; struct lttng_bytecode_node { - enum lttng_bytecode_node_type type; + enum lttng_kernel_bytecode_type type; struct list_head node; struct lttng_enabler *enabler; struct { @@ -393,29 +393,16 @@ struct lttng_bytecode_node { } bc; }; -/* - * Bytecode interpreter return value masks. - */ -enum lttng_bytecode_interpreter_ret { - LTTNG_INTERPRETER_DISCARD = 0, - LTTNG_INTERPRETER_RECORD_FLAG = (1ULL << 0), - /* Other bits are kept for future use. */ -}; - struct lttng_interpreter_output; struct lttng_bytecode_runtime { /* Associated bytecode */ + enum lttng_kernel_bytecode_type type; struct lttng_bytecode_node *bc; - union { - uint64_t (*filter)(void *filter_data, - struct lttng_probe_ctx *lttng_probe_ctx, - const char *filter_stack_data); - uint64_t (*capture)(void *filter_data, - const char *capture_stack_data, + int (*interpreter_func)(struct lttng_bytecode_runtime *kernel_bytecode, + const char *interpreter_stack_data, struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output); - } interpreter_funcs; + void *caller_ctx); int link_failed; struct list_head node; /* list of bytecode runtime in event */ struct lttng_kernel_ctx *ctx; @@ -456,6 +443,14 @@ enum lttng_syscall_abi { LTTNG_SYSCALL_ABI_COMPAT, }; +/* + * Result of the run_filter() callback. + */ +enum lttng_kernel_event_filter_result { + LTTNG_KERNEL_EVENT_FILTER_ACCEPT = 0, + LTTNG_KERNEL_EVENT_FILTER_REJECT = 1, +}; + struct lttng_kernel_event_common_private; enum lttng_kernel_event_type { @@ -471,8 +466,9 @@ struct lttng_kernel_event_common { int enabled; int eval_filter; /* Need to evaluate filters */ - int (*run_filter)(struct lttng_kernel_event_common *event, + int (*run_filter)(const struct lttng_kernel_event_common *event, const char *stack_data, + struct lttng_probe_ctx *probe_ctx, void *filter_ctx); }; diff --git a/include/lttng/lttng-bytecode.h b/include/lttng/lttng-bytecode.h index d2d5fea1..6313b2cc 100644 --- a/include/lttng/lttng-bytecode.h +++ b/include/lttng/lttng-bytecode.h @@ -284,26 +284,19 @@ struct lttng_interpreter_output { const char *lttng_bytecode_print_op(enum bytecode_op op); -void lttng_bytecode_filter_sync_state(struct lttng_bytecode_runtime *runtime); -void lttng_bytecode_capture_sync_state(struct lttng_bytecode_runtime *runtime); +void lttng_bytecode_sync_state(struct lttng_bytecode_runtime *runtime); int lttng_bytecode_validate(struct bytecode_runtime *bytecode); int lttng_bytecode_specialize(const struct lttng_kernel_event_desc *event_desc, struct bytecode_runtime *bytecode); -uint64_t lttng_bytecode_filter_interpret_false(void *filter_data, - struct lttng_probe_ctx *lttng_probe_ctx, - const char *filter_stack_data); -uint64_t lttng_bytecode_filter_interpret(void *filter_data, - struct lttng_probe_ctx *lttng_probe_ctx, - const char *filter_stack_data); +int lttng_bytecode_interpret_error(struct lttng_bytecode_runtime *bytecode_runtime, + const char *stack_data, + struct lttng_probe_ctx *probe_ctx, + void *ctx); -uint64_t lttng_bytecode_capture_interpret_false(void *capture_data, - const char *capture_stack_data, - struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output); -uint64_t lttng_bytecode_capture_interpret(void *capture_data, - const char *capture_stack_data, +int lttng_bytecode_interpret(struct lttng_bytecode_runtime *kernel_bytecode, + const char *interpreter_stack_data, struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output); + void *caller_ctx); #endif /* _LTTNG_FILTER_H */ diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index d70c1f97..5af11ba7 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -1068,20 +1068,11 @@ static void __event_probe__##_name(_data_proto) \ __dynamic_len_idx = __orig_dynamic_len_offset; \ _code_pre \ if (unlikely(READ_ONCE(__event->eval_filter))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - bool __filter_record = false; \ - \ __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ _locvar_args); \ __interpreter_stack_prepared = true; \ - lttng_list_for_each_entry_rcu(bc_runtime, &__event->priv->filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \ - __filter_record = true; \ - break; \ - } \ - } \ - if (likely(!__filter_record)) \ + if (likely(__event->run_filter(__event, \ + __stackvar.__interpreter_stack_data, &__lttng_probe_ctx, NULL) != LTTNG_KERNEL_EVENT_FILTER_ACCEPT)) \ goto __post; \ } \ switch (__event->type) { \ diff --git a/src/lttng-bytecode-interpreter.c b/src/lttng-bytecode-interpreter.c index ca836d15..0161d673 100644 --- a/src/lttng-bytecode-interpreter.c +++ b/src/lttng-bytecode-interpreter.c @@ -208,19 +208,13 @@ int stack_strcmp(struct estack *stack, int top, const char *cmp_type) return diff; } -uint64_t lttng_bytecode_filter_interpret_false(void *filter_data, - struct lttng_probe_ctx *lttng_probe_ctx, - const char *filter_stack_data) -{ - return LTTNG_INTERPRETER_DISCARD; -} - -uint64_t lttng_bytecode_capture_interpret_false(void *filter_data, - const char *capture_stack_data, - struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output) +int lttng_bytecode_interpret_error( + struct lttng_bytecode_runtime *bytecode_runtime __attribute__((unused)), + const char *stack_data __attribute__((unused)), + struct lttng_probe_ctx *probe_ctx __attribute__((unused)), + void *ctx __attribute__((unused))) { - return LTTNG_INTERPRETER_DISCARD; + return LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR; } #ifdef INTERPRETER_USE_SWITCH @@ -717,7 +711,7 @@ again: return -EINVAL; } - return LTTNG_INTERPRETER_RECORD_FLAG; + return 0; } #ifdef DEBUG @@ -754,17 +748,20 @@ void dbg_load_ref_user_str_printk(const struct estack_entry *user_str_reg) #endif /* - * Return 0 (discard), or raise the 0x1 flag (log event). - * Currently, other flags are kept for future extensions and have no - * effect. + * Return LTTNG_KERNEL_BYTECODE_INTERPRETER_OK on success. + * Return LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR on error. + * + * For FILTER bytecode: expect a struct lttng_kernel_bytecode_filter_ctx * + * as @ctx argument. + * For CAPTURE bytecode: expect a struct lttng_interpreter_output * + * as @ctx argument. */ -static -uint64_t bytecode_interpret(void *interpreter_data, +int lttng_bytecode_interpret(struct lttng_bytecode_runtime *kernel_bytecode, const char *interpreter_stack_data, struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output) + void *caller_ctx) { - struct bytecode_runtime *bytecode = interpreter_data; + struct bytecode_runtime *bytecode = container_of(kernel_bytecode, struct bytecode_runtime, p); void *pc, *next_pc, *start_pc; int ret = -EINVAL; uint64_t retval = 0; @@ -928,8 +925,7 @@ uint64_t bytecode_interpret(void *interpreter_data, goto end; OP(BYTECODE_OP_RETURN): - OP(BYTECODE_OP_RETURN_S64): - /* LTTNG_INTERPRETER_DISCARD or LTTNG_INTERPRETER_RECORD_FLAG */ + /* LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR or LTTNG_KERNEL_BYTECODE_INTERPRETER_OK */ switch (estack_ax_t) { case REG_S64: case REG_U64: @@ -938,7 +934,7 @@ uint64_t bytecode_interpret(void *interpreter_data, case REG_DOUBLE: case REG_STRING: case REG_PTR: - if (!output) { + if (kernel_bytecode->type != LTTNG_KERNEL_BYTECODE_TYPE_CAPTURE) { ret = -EINVAL; goto end; } @@ -952,6 +948,12 @@ uint64_t bytecode_interpret(void *interpreter_data, ret = 0; goto end; + OP(BYTECODE_OP_RETURN_S64): + /* LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR or LTTNG_KERNEL_BYTECODE_INTERPRETER_OK */ + retval = !!estack_ax_v; + ret = 0; + goto end; + /* binary */ OP(BYTECODE_OP_MUL): OP(BYTECODE_OP_DIV): @@ -1811,34 +1813,63 @@ uint64_t bytecode_interpret(void *interpreter_data, END_OP end: - /* Return _DISCARD on error. */ + /* No need to prepare output if an error occurred. */ if (ret) - return LTTNG_INTERPRETER_DISCARD; + return LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR; - if (output) { - return lttng_bytecode_interpret_format_output( - estack_ax(stack, top), output); + /* Prepare output. */ + switch (kernel_bytecode->type) { + case LTTNG_KERNEL_BYTECODE_TYPE_FILTER: + { + struct lttng_kernel_bytecode_filter_ctx *filter_ctx = + (struct lttng_kernel_bytecode_filter_ctx *) caller_ctx; + if (retval) + filter_ctx->result = LTTNG_KERNEL_BYTECODE_FILTER_ACCEPT; + else + filter_ctx->result = LTTNG_KERNEL_BYTECODE_FILTER_REJECT; + break; } - - return retval; -} -LTTNG_STACK_FRAME_NON_STANDARD(bytecode_interpret); - -uint64_t lttng_bytecode_filter_interpret(void *filter_data, - struct lttng_probe_ctx *lttng_probe_ctx, - const char *filter_stack_data) -{ - return bytecode_interpret(filter_data, - filter_stack_data, lttng_probe_ctx, NULL); + case LTTNG_KERNEL_BYTECODE_TYPE_CAPTURE: + ret = lttng_bytecode_interpret_format_output(estack_ax(stack, top), + (struct lttng_interpreter_output *) caller_ctx); + break; + default: + ret = -EINVAL; + break; + } + if (ret) + return LTTNG_KERNEL_BYTECODE_INTERPRETER_ERROR; + else + return LTTNG_KERNEL_BYTECODE_INTERPRETER_OK; } +LTTNG_STACK_FRAME_NON_STANDARD(lttng_bytecode_interpret); -uint64_t lttng_bytecode_capture_interpret(void *capture_data, - const char *capture_stack_data, - struct lttng_probe_ctx *lttng_probe_ctx, - struct lttng_interpreter_output *output) +/* + * Return LTTNG_KERNEL_EVENT_FILTER_ACCEPT or LTTNG_KERNEL_EVENT_FILTER_REJECT. + */ +int lttng_kernel_interpret_event_filter(const struct lttng_kernel_event_common *event, + const char *interpreter_stack_data, + struct lttng_probe_ctx *probe_ctx, + void *event_filter_ctx __attribute__((unused))) { - return bytecode_interpret(capture_data, - capture_stack_data, lttng_probe_ctx, output); + struct lttng_bytecode_runtime *filter_bc_runtime; + struct list_head *filter_bytecode_runtime_head = &event->priv->filter_bytecode_runtime_head; + struct lttng_kernel_bytecode_filter_ctx bytecode_filter_ctx; + bool filter_record = false; + + list_for_each_entry_rcu(filter_bc_runtime, filter_bytecode_runtime_head, node) { + if (likely(filter_bc_runtime->interpreter_func(filter_bc_runtime, + interpreter_stack_data, probe_ctx, &bytecode_filter_ctx) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)) { + if (unlikely(bytecode_filter_ctx.result == LTTNG_KERNEL_BYTECODE_FILTER_ACCEPT)) { + filter_record = true; + break; + } + } + } + if (filter_record) + return LTTNG_KERNEL_EVENT_FILTER_ACCEPT; + else + return LTTNG_KERNEL_EVENT_FILTER_REJECT; } #undef START_OP diff --git a/src/lttng-bytecode.c b/src/lttng-bytecode.c index 5e4f8b3c..e1d0afe6 100644 --- a/src/lttng-bytecode.c +++ b/src/lttng-bytecode.c @@ -450,6 +450,7 @@ int link_bytecode(const struct lttng_kernel_event_desc *event_desc, ret = -ENOMEM; goto alloc_error; } + runtime->p.type = bytecode->type; runtime->p.bc = bytecode; runtime->p.ctx = ctx; runtime->len = bytecode->bc.reloc_offset; @@ -483,18 +484,7 @@ int link_bytecode(const struct lttng_kernel_event_desc *event_desc, if (ret) { goto link_error; } - - switch (bytecode->type) { - case LTTNG_BYTECODE_NODE_TYPE_FILTER: - runtime->p.interpreter_funcs.filter = lttng_bytecode_filter_interpret; - break; - case LTTNG_BYTECODE_NODE_TYPE_CAPTURE: - runtime->p.interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; - break; - default: - WARN_ON(1); - } - + runtime->p.interpreter_func = lttng_bytecode_interpret; runtime->p.link_failed = 0; list_add_rcu(&runtime->p.node, insert_loc); dbg_printk("Linking successful.\n"); @@ -502,16 +492,7 @@ int link_bytecode(const struct lttng_kernel_event_desc *event_desc, link_error: - switch (bytecode->type) { - case LTTNG_BYTECODE_NODE_TYPE_FILTER: - runtime->p.interpreter_funcs.filter = lttng_bytecode_filter_interpret_false; - break; - case LTTNG_BYTECODE_NODE_TYPE_CAPTURE: - runtime->p.interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; - break; - default: - WARN_ON(1); - } + runtime->p.interpreter_func = lttng_bytecode_interpret_error; runtime->p.link_failed = 1; list_add_rcu(&runtime->p.node, insert_loc); alloc_error: @@ -519,24 +500,14 @@ alloc_error: return ret; } -void lttng_bytecode_filter_sync_state(struct lttng_bytecode_runtime *runtime) -{ - struct lttng_bytecode_node *bc = runtime->bc; - - if (!bc->enabler->enabled || runtime->link_failed) - runtime->interpreter_funcs.filter = lttng_bytecode_filter_interpret_false; - else - runtime->interpreter_funcs.filter = lttng_bytecode_filter_interpret; -} - -void lttng_bytecode_capture_sync_state(struct lttng_bytecode_runtime *runtime) +void lttng_bytecode_sync_state(struct lttng_bytecode_runtime *runtime) { struct lttng_bytecode_node *bc = runtime->bc; if (!bc->enabler->enabled || runtime->link_failed) - runtime->interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; + runtime->interpreter_func = lttng_bytecode_interpret_error; else - runtime->interpreter_funcs.capture = lttng_bytecode_capture_interpret; + runtime->interpreter_func = lttng_bytecode_interpret; } /* diff --git a/src/lttng-event-notifier-notification.c b/src/lttng-event-notifier-notification.c index 2286aa79..ce5a670f 100644 --- a/src/lttng-event-notifier-notification.c +++ b/src/lttng-event-notifier-notification.c @@ -400,8 +400,8 @@ void notification_send(struct lttng_event_notifier_notification *notif, reserve_size += capture_buffer_content_len; kernel_notif.capture_buf_size = capture_buffer_content_len; - lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, NULL, reserve_size, - lttng_alignof(kernel_notif), -1); + lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size, + lttng_alignof(kernel_notif), -1, NULL); ret = event_notifier_group->ops->event_reserve(&ctx, 0); if (ret < 0) { record_error(event_notifier); @@ -455,8 +455,8 @@ void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier * &event_notifier->priv->capture_bytecode_runtime_head, node) { struct lttng_interpreter_output output; - if (capture_bc_runtime->interpreter_funcs.capture(capture_bc_runtime, - stack_data, probe_ctx, &output) & LTTNG_INTERPRETER_RECORD_FLAG) + if (capture_bc_runtime->interpreter_func(capture_bc_runtime, + stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK) ret = notification_append_capture(¬if, &output); else ret = notification_append_empty_capture(¬if); diff --git a/src/lttng-events.c b/src/lttng-events.c index 0a36ccb3..d881bddc 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -885,6 +885,7 @@ struct lttng_kernel_event_recorder *_lttng_kernel_event_recorder_create(struct l event_recorder->parent.priv = &event_recorder_priv->parent; event_recorder->parent.type = LTTNG_KERNEL_EVENT_TYPE_RECORDER; + event_recorder->parent.run_filter = lttng_kernel_interpret_event_filter; event_recorder->chan = chan; event_recorder->priv->id = chan->free_event_id++; event_recorder->priv->parent.instrumentation = itype; @@ -960,6 +961,7 @@ struct lttng_kernel_event_recorder *_lttng_kernel_event_recorder_create(struct l event_recorder_return->parent.priv = &event_recorder_return_priv->parent; event_recorder_return->parent.type = LTTNG_KERNEL_EVENT_TYPE_RECORDER; + event_recorder_return->parent.run_filter = lttng_kernel_interpret_event_filter; event_recorder_return->chan = chan; event_recorder_return->priv->id = chan->free_event_id++; event_recorder_return->priv->parent.instrumentation = itype; @@ -1161,6 +1163,7 @@ struct lttng_kernel_event_notifier *_lttng_event_notifier_create( INIT_LIST_HEAD(&event_notifier->priv->parent.filter_bytecode_runtime_head); INIT_LIST_HEAD(&event_notifier->priv->parent.enablers_ref_head); INIT_LIST_HEAD(&event_notifier->priv->capture_bytecode_runtime_head); + event_notifier->parent.run_filter = lttng_kernel_interpret_event_filter; switch (itype) { case LTTNG_KERNEL_ABI_TRACEPOINT: @@ -2462,7 +2465,7 @@ int lttng_enabler_attach_filter_bytecode(struct lttng_enabler *enabler, if (ret) goto error_free; - bytecode_node->type = LTTNG_BYTECODE_NODE_TYPE_FILTER; + bytecode_node->type = LTTNG_KERNEL_BYTECODE_TYPE_FILTER; bytecode_node->enabler = enabler; /* Enforce length based on allocated size */ bytecode_node->bc.len = bytecode_len; @@ -2621,7 +2624,7 @@ int lttng_event_notifier_enabler_attach_capture_bytecode( if (ret) goto error_free; - bytecode_node->type = LTTNG_BYTECODE_NODE_TYPE_CAPTURE; + bytecode_node->type = LTTNG_KERNEL_BYTECODE_TYPE_CAPTURE; bytecode_node->enabler = enabler; /* Enforce length based on allocated size */ @@ -2728,7 +2731,7 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) /* Enable filters */ list_for_each_entry(runtime, &event_recorder_priv->parent.filter_bytecode_runtime_head, node) { - lttng_bytecode_filter_sync_state(runtime); + lttng_bytecode_sync_state(runtime); nr_filters++; } WRITE_ONCE(event_recorder_priv->parent.pub->eval_filter, @@ -2818,7 +2821,7 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group /* Enable filters */ list_for_each_entry(runtime, &event_notifier_priv->parent.filter_bytecode_runtime_head, node) { - lttng_bytecode_filter_sync_state(runtime); + lttng_bytecode_sync_state(runtime); nr_filters++; } WRITE_ONCE(event_notifier_priv->parent.pub->eval_filter, @@ -2827,7 +2830,7 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group /* Enable captures */ list_for_each_entry(runtime, &event_notifier_priv->capture_bytecode_runtime_head, node) { - lttng_bytecode_capture_sync_state(runtime); + lttng_bytecode_sync_state(runtime); nr_captures++; } WRITE_ONCE(event_notifier->eval_capture, !!nr_captures);