1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * lttng/tracepoint-event-impl.h
5 * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
6 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 #include <linux/uaccess.h>
10 #include <linux/debugfs.h>
11 #include <linux/rculist.h>
12 #include <asm/byteorder.h>
13 #include <linux/swab.h>
15 #include <wrapper/vmalloc.h> /* for wrapper_vmalloc_sync_mappings() */
16 #include <ringbuffer/frontend_types.h>
17 #include <ringbuffer/backend.h>
18 #include <wrapper/rcu.h>
19 #include <wrapper/user_namespace.h>
20 #include <lttng/types.h>
21 #include <lttng/probe-user.h>
22 #include <lttng/events.h>
23 #include <lttng/tracer-core.h>
24 #include <lttng/tp-mempool.h>
26 #define __LTTNG_NULL_STRING "(null)"
29 #define PARAMS(args...) args
32 * Macro declarations used for all stages.
36 * LTTng name mapping macros. LTTng remaps some of the kernel events to
37 * enforce name-spacing.
39 #undef LTTNG_TRACEPOINT_EVENT_MAP
40 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
41 LTTNG_TRACEPOINT_EVENT_CLASS(map, \
45 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
47 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
48 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields) \
49 LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map, \
51 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
53 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
54 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
55 LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map, \
62 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
64 #undef LTTNG_TRACEPOINT_EVENT_CODE
65 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
66 LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name, \
75 * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
76 * handlers for events. That is, if all events have the same parameters
77 * and just have distinct trace points. Each tracepoint can be defined
78 * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
79 * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
81 * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
85 #undef LTTNG_TRACEPOINT_EVENT
86 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields) \
87 LTTNG_TRACEPOINT_EVENT_MAP(name, name, \
92 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
93 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields) \
94 LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
96 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
97 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args) \
98 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
100 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
101 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name) \
102 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
104 #undef LTTNG_TRACEPOINT_EVENT_CLASS
105 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
106 LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
109 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
110 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
111 LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
115 * Stage 1 of the trace events.
117 * Create dummy trace calls for each events, verifying that the LTTng module
118 * instrumentation headers match the kernel arguments. Will be optimized
119 * out by the compiler.
122 /* Reset all macros within TRACEPOINT_EVENT */
123 #include <lttng/events-reset.h>
126 #define TP_PROTO(...) __VA_ARGS__
129 #define TP_ARGS(...) __VA_ARGS__
131 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
132 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
133 void trace_##_name(_proto);
135 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
136 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
137 void trace_##_name(void);
139 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
142 * Stage 1.1 of the trace events.
144 * Create dummy trace prototypes for each event class, and for each used
145 * template. This will allow checking whether the prototypes from the
146 * class and the instance using the class actually match.
149 #include <lttng/events-reset.h> /* Reset all macros within TRACE_EVENT */
152 #define TP_PROTO(...) __VA_ARGS__
155 #define TP_ARGS(...) __VA_ARGS__
157 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
158 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
159 void __event_template_proto___##_template(_proto);
161 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
162 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
163 void __event_template_proto___##_template(void);
165 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
166 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
167 void __event_template_proto___##_name(_proto);
169 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
170 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
171 void __event_template_proto___##_name(void);
173 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
176 * Stage 1.2 of the trace event_notifier.
178 * Create dummy trace prototypes for each event class, and for each used
179 * template. This will allow checking whether the prototypes from the
180 * class and the instance using the class actually match.
183 #include <lttng/events-reset.h> /* Reset all macros within TRACE_EVENT */
186 #define TP_PROTO(...) __VA_ARGS__
189 #define TP_ARGS(...) __VA_ARGS__
191 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
192 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
193 void __event_notifier_template_proto___##_template(_proto);
195 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
196 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
197 void __event_notifier_template_proto___##_template(void);
199 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
200 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
201 void __event_notifier_template_proto___##_name(_proto);
203 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
204 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
205 void __event_notifier_template_proto___##_name(void);
207 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
211 * Stage 1.2 of tracepoint event generation
213 * Unfolding the enums
215 #include <lttng/events-reset.h> /* Reset all macros within TRACE_EVENT */
217 /* Enumeration entry (single value) */
218 #undef ctf_enum_value
219 #define ctf_enum_value(_string, _value) \
220 lttng_kernel_static_enum_entry_value(_string, _value)
222 /* Enumeration entry (range) */
223 #undef ctf_enum_range
224 #define ctf_enum_range(_string, _range_start, _range_end) \
225 lttng_kernel_static_enum_entry_range(_string, _range_start, _range_end)
227 /* Enumeration entry (automatic value; follows the rules of CTF) */
229 #define ctf_enum_auto(_string) \
230 lttng_kernel_static_enum_entry_auto(_string)
232 #undef TP_ENUM_VALUES
233 #define TP_ENUM_VALUES(...) \
236 #undef LTTNG_TRACEPOINT_ENUM
237 #define LTTNG_TRACEPOINT_ENUM(_name, _values) \
238 static const struct lttng_kernel_enum_entry *__enum_values__##_name[] = { \
242 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
245 * Stage 2 of the trace events.
247 * Create event field type metadata section.
248 * Each event produce an array of fields.
251 /* Reset all macros within TRACEPOINT_EVENT */
252 #include <lttng/events-reset.h>
253 #include <lttng/events-write.h>
254 #include <lttng/events-nowrite.h>
256 #undef _ctf_integer_ext
257 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
258 lttng_kernel_static_event_field(#_item, \
259 lttng_kernel_static_type_integer_from_type(_type, _byte_order, _base), \
262 #undef _ctf_array_encoded
263 #define _ctf_array_encoded(_type, _item, _src, _length, \
264 _encoding, _byte_order, _elem_type_base, _user, _nowrite) \
265 lttng_kernel_static_event_field(#_item, \
266 lttng_kernel_static_type_array(_length, \
267 lttng_kernel_static_type_integer_from_type(_type, _byte_order, _elem_type_base), \
272 #undef _ctf_array_bitfield
273 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
274 lttng_kernel_static_event_field(#_item, \
275 lttng_kernel_static_type_array((_length) * sizeof(_type) * CHAR_BIT, \
276 lttng_kernel_static_type_integer(1, 1, 0, __LITTLE_ENDIAN, 10), \
277 lttng_alignof(_type), \
281 #undef _ctf_sequence_encoded
282 #define _ctf_sequence_encoded(_type, _item, _src, \
283 _length_type, _src_length, _encoding, \
284 _byte_order, _elem_type_base, _user, _nowrite) \
285 lttng_kernel_static_event_field("_" #_item "_length", \
286 lttng_kernel_static_type_integer_from_type(_length_type, __BYTE_ORDER, 10), \
288 lttng_kernel_static_event_field(#_item, \
289 lttng_kernel_static_type_sequence("_" #_item "_length", \
290 lttng_kernel_static_type_integer_from_type(_type, _byte_order, _elem_type_base), \
295 #undef _ctf_sequence_bitfield
296 #define _ctf_sequence_bitfield(_type, _item, _src, \
297 _length_type, _src_length, \
299 lttng_kernel_static_event_field("_" #_item "_length", \
300 lttng_kernel_static_type_integer_from_type(_length_type, __BYTE_ORDER, 10), \
302 lttng_kernel_static_event_field(#_item, \
303 lttng_kernel_static_type_sequence("_" #_item "_length", \
304 lttng_kernel_static_type_integer(1, 1, 0, __LITTLE_ENDIAN, 10), \
305 lttng_alignof(_type), \
310 #define _ctf_string(_item, _src, _user, _nowrite) \
311 lttng_kernel_static_event_field(#_item, \
312 lttng_kernel_static_type_string(UTF8), \
316 #define _ctf_unused(_src)
319 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite) \
320 lttng_kernel_static_event_field(#_item, \
321 lttng_kernel_static_type_enum(&__enum_##_name, \
322 lttng_kernel_static_type_integer_from_type(_type, __BYTE_ORDER, 10)), \
325 #undef ctf_custom_field
326 #define ctf_custom_field(_type, _item, _code) \
327 lttng_kernel_static_event_field(#_item, PARAMS(_type), 0, 0, 1),
329 #undef ctf_custom_type
330 #define ctf_custom_type(...) __VA_ARGS__
333 #define TP_FIELDS(...) __VA_ARGS__ /* Only one used in this phase */
335 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
336 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
337 static const struct lttng_kernel_event_field *__event_fields___##_name[] = { \
341 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
342 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
343 LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
345 #undef LTTNG_TRACEPOINT_ENUM
346 #define LTTNG_TRACEPOINT_ENUM(_name, _values) \
347 static const struct lttng_kernel_enum_desc __enum_##_name = { \
349 .entries = __enum_values__##_name, \
350 .nr_entries = ARRAY_SIZE(__enum_values__##_name), \
353 #define LTTNG_CREATE_FIELD_METADATA
354 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
355 #undef LTTNG_CREATE_FIELD_METADATA
358 * Stage 3 of the trace events.
360 * Create probe callback prototypes.
363 /* Reset all macros within TRACEPOINT_EVENT */
364 #include <lttng/events-reset.h>
367 #define TP_PROTO(...) __VA_ARGS__
369 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
370 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
371 static void __event_probe__##_name(void *__data, _proto);
373 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
374 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
375 static void __event_probe__##_name(void *__data);
377 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
380 * Stage 3.1 of the trace event_notifiers.
382 * Create event_notifier probe callback prototypes.
385 /* Reset all macros within TRACEPOINT_EVENT */
386 #include <lttng/events-reset.h>
389 #define TP_PROTO(...) __VA_ARGS__
391 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
392 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
393 static void __event_notifier_probe__##_name(void *__data, _proto);
395 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
396 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
397 static void __event_notifier_probe__##_name(void *__data);
399 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
402 * Stage 4 of the trace events.
404 * Create static inline function that calculates event size.
407 /* Reset all macros within TRACEPOINT_EVENT */
408 #include <lttng/events-reset.h>
409 #include <lttng/events-write.h>
411 #undef _ctf_integer_ext
412 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
413 __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
414 __event_len += sizeof(_type);
416 #undef _ctf_array_encoded
417 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
418 __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
419 __event_len += sizeof(_type) * (_length);
421 #undef _ctf_array_bitfield
422 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
423 _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
425 #undef _ctf_sequence_encoded
426 #define _ctf_sequence_encoded(_type, _item, _src, _length_type, \
427 _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
428 __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
429 __event_len += sizeof(_length_type); \
430 __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
432 size_t __seqlen = (_src_length); \
434 if (unlikely(++this_cpu_ptr(<tng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
436 barrier(); /* reserve before use. */ \
437 this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = __seqlen; \
438 __event_len += sizeof(_type) * __seqlen; \
441 #undef _ctf_sequence_bitfield
442 #define _ctf_sequence_bitfield(_type, _item, _src, \
443 _length_type, _src_length, \
445 _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
446 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
449 * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
453 #define _ctf_string(_item, _src, _user, _nowrite) \
454 if (unlikely(++this_cpu_ptr(<tng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
456 barrier(); /* reserve before use. */ \
458 __event_len += this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = \
459 max_t(size_t, lttng_strlen_user_inatomic(_src), 1); \
461 __event_len += this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = \
462 strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
466 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite) \
467 _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
470 #define ctf_align(_type) \
471 __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
473 #undef ctf_custom_field
474 #define ctf_custom_field(_type, _item, _code) \
479 #undef ctf_custom_code
480 #define ctf_custom_code(...) __VA_ARGS__
483 #define TP_PROTO(...) __VA_ARGS__
486 #define TP_FIELDS(...) __VA_ARGS__
489 #define TP_locvar(...) __VA_ARGS__
491 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
492 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
493 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto) \
495 size_t __event_len = 0; \
496 unsigned int __dynamic_len_idx __attribute__((unused)) = 0; \
497 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
500 return __event_len; \
503 __attribute__((unused)); \
507 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
508 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
509 static inline ssize_t __event_get_size__##_name(void *__tp_locvar) \
511 size_t __event_len = 0; \
512 unsigned int __dynamic_len_idx __attribute__((unused)) = 0; \
513 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
516 return __event_len; \
519 __attribute__((unused)); \
523 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
527 * Stage 4.1 of tracepoint event generation.
529 * Create static inline function that layout the filter stack data.
530 * We make both write and nowrite data available to the filter.
533 /* Reset all macros within TRACEPOINT_EVENT */
534 #include <lttng/events-reset.h>
535 #include <lttng/events-write.h>
536 #include <lttng/events-nowrite.h>
538 #undef _ctf_integer_ext_fetched
539 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
540 if (lttng_is_signed_type(_type)) { \
541 int64_t __ctf_tmp_int64; \
542 switch (sizeof(_type)) { \
545 union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
546 __ctf_tmp_int64 = (int64_t) __tmp.v; \
551 union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
552 if (_byte_order != __BYTE_ORDER) \
553 __swab16s(&__tmp.v); \
554 __ctf_tmp_int64 = (int64_t) __tmp.v; \
559 union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
560 if (_byte_order != __BYTE_ORDER) \
561 __swab32s(&__tmp.v); \
562 __ctf_tmp_int64 = (int64_t) __tmp.v; \
567 union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
568 if (_byte_order != __BYTE_ORDER) \
569 __swab64s(&__tmp.v); \
570 __ctf_tmp_int64 = (int64_t) __tmp.v; \
576 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t)); \
578 uint64_t __ctf_tmp_uint64; \
579 switch (sizeof(_type)) { \
582 union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
583 __ctf_tmp_uint64 = (uint64_t) __tmp.v; \
588 union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
589 if (_byte_order != __BYTE_ORDER) \
590 __swab16s(&__tmp.v); \
591 __ctf_tmp_uint64 = (uint64_t) __tmp.v; \
596 union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
597 if (_byte_order != __BYTE_ORDER) \
598 __swab32s(&__tmp.v); \
599 __ctf_tmp_uint64 = (uint64_t) __tmp.v; \
604 union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
605 if (_byte_order != __BYTE_ORDER) \
606 __swab64s(&__tmp.v); \
607 __ctf_tmp_uint64 = (uint64_t) __tmp.v; \
613 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t)); \
615 __stack_data += sizeof(int64_t);
617 #undef _ctf_integer_ext_isuser0
618 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
619 _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
621 #undef _ctf_integer_ext_isuser1
622 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
625 char __array[sizeof(_user_src)]; \
626 __typeof__(_user_src) __v; \
628 if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array, \
629 &(_user_src), sizeof(_user_src))) \
630 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array)); \
631 _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
634 #undef _ctf_integer_ext
635 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
636 _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
638 #undef _ctf_array_encoded
639 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
641 unsigned long __ctf_tmp_ulong = (unsigned long) (_length); \
642 const void *__ctf_tmp_ptr = (_src); \
643 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
644 __stack_data += sizeof(unsigned long); \
645 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *)); \
646 __stack_data += sizeof(void *); \
649 #undef _ctf_array_bitfield
650 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
651 _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
653 #undef _ctf_sequence_encoded
654 #define _ctf_sequence_encoded(_type, _item, _src, _length_type, \
655 _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
657 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
658 const void *__ctf_tmp_ptr = (_src); \
659 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
660 __stack_data += sizeof(unsigned long); \
661 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *)); \
662 __stack_data += sizeof(void *); \
665 #undef _ctf_sequence_bitfield
666 #define _ctf_sequence_bitfield(_type, _item, _src, \
667 _length_type, _src_length, \
669 _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
670 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
673 #define _ctf_string(_item, _src, _user, _nowrite) \
675 const void *__ctf_tmp_ptr = \
676 ((_src) ? (_src) : __LTTNG_NULL_STRING); \
677 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *)); \
678 __stack_data += sizeof(void *); \
682 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite) \
683 _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
686 #define TP_PROTO(...) __VA_ARGS__
689 #define TP_FIELDS(...) __VA_ARGS__
692 #define TP_locvar(...) __VA_ARGS__
694 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
695 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
697 void __event_prepare_interpreter_stack__##_name(char *__stack_data, \
700 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
705 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
706 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
708 void __event_prepare_interpreter_stack__##_name(char *__stack_data, \
709 void *__tp_locvar, _proto) \
711 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
716 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
719 * Stage 5 of the trace events.
721 * Create static inline function that calculates event payload alignment.
724 /* Reset all macros within TRACEPOINT_EVENT */
725 #include <lttng/events-reset.h>
726 #include <lttng/events-write.h>
728 #undef _ctf_integer_ext
729 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
730 __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
732 #undef _ctf_array_encoded
733 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
734 __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
736 #undef _ctf_array_bitfield
737 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
738 _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
740 #undef _ctf_sequence_encoded
741 #define _ctf_sequence_encoded(_type, _item, _src, _length_type, \
742 _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
743 __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
744 __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
746 #undef _ctf_sequence_bitfield
747 #define _ctf_sequence_bitfield(_type, _item, _src, \
748 _length_type, _src_length, \
750 _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
751 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
754 #define _ctf_string(_item, _src, _user, _nowrite)
757 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite) \
758 _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
761 #define ctf_align(_type) \
762 __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
765 #define TP_PROTO(...) __VA_ARGS__
768 #define TP_FIELDS(...) __VA_ARGS__
771 #define TP_locvar(...) __VA_ARGS__
773 #undef ctf_custom_field
774 #define ctf_custom_field(_type, _item, _code) _code
776 #undef ctf_custom_code
777 #define ctf_custom_code(...) \
782 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
783 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
784 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto) \
786 size_t __event_align = 1; \
787 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
790 return __event_align; \
793 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
794 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
795 static inline size_t __event_get_align__##_name(void *__tp_locvar) \
797 size_t __event_align = 1; \
798 struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \
801 return __event_align; \
804 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
807 * Stage 6 of tracepoint event generation.
809 * Create the probe function. This function calls event size calculation
810 * and writes event data into the buffer.
813 /* Reset all macros within TRACEPOINT_EVENT */
814 #include <lttng/events-reset.h>
815 #include <lttng/events-write.h>
817 #undef _ctf_integer_ext_fetched
818 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
820 _type __tmp = _src; \
821 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
822 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
825 #undef _ctf_integer_ext_isuser0
826 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
827 _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
829 #undef _ctf_integer_ext_isuser1
830 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
833 char __array[sizeof(_user_src)]; \
834 __typeof__(_user_src) __v; \
836 if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array, \
837 &(_user_src), sizeof(_user_src))) \
838 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array)); \
839 _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
842 #undef _ctf_integer_ext
843 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
844 _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
846 #undef _ctf_array_encoded
847 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
848 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
850 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
852 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
855 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
856 #undef _ctf_array_bitfield
857 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
858 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
860 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
862 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
864 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
866 * For big endian, we need to byteswap into little endian.
868 #undef _ctf_array_bitfield
869 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
870 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
874 for (_i = 0; _i < (_length); _i++) { \
878 if (get_user(_tmp, (_type *) _src + _i)) \
881 _tmp = ((_type *) _src)[_i]; \
883 switch (sizeof(_type)) { \
887 _tmp = cpu_to_le16(_tmp); \
890 _tmp = cpu_to_le32(_tmp); \
893 _tmp = cpu_to_le64(_tmp); \
898 __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
901 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
903 #undef _ctf_sequence_encoded
904 #define _ctf_sequence_encoded(_type, _item, _src, _length_type, \
905 _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
907 _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
908 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
909 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
911 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
913 __chan->ops->event_write_from_user(&__ctx, _src, \
914 sizeof(_type) * __get_dynamic_len(dest)); \
916 __chan->ops->event_write(&__ctx, _src, \
917 sizeof(_type) * __get_dynamic_len(dest)); \
920 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
921 #undef _ctf_sequence_bitfield
922 #define _ctf_sequence_bitfield(_type, _item, _src, \
923 _length_type, _src_length, \
926 _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
927 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
928 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
930 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
932 __chan->ops->event_write_from_user(&__ctx, _src, \
933 sizeof(_type) * __get_dynamic_len(dest)); \
935 __chan->ops->event_write(&__ctx, _src, \
936 sizeof(_type) * __get_dynamic_len(dest)); \
938 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
940 * For big endian, we need to byteswap into little endian.
942 #undef _ctf_sequence_bitfield
943 #define _ctf_sequence_bitfield(_type, _item, _src, \
944 _length_type, _src_length, \
947 _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
948 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
949 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
951 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type)); \
953 size_t _i, _length; \
955 _length = __get_dynamic_len(dest); \
956 for (_i = 0; _i < _length; _i++) { \
960 if (get_user(_tmp, (_type *) _src + _i)) \
963 _tmp = ((_type *) _src)[_i]; \
965 switch (sizeof(_type)) { \
969 _tmp = cpu_to_le16(_tmp); \
972 _tmp = cpu_to_le32(_tmp); \
975 _tmp = cpu_to_le64(_tmp); \
980 __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
983 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
986 #define _ctf_string(_item, _src, _user, _nowrite) \
988 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
989 __chan->ops->event_strcpy_from_user(&__ctx, _src, \
990 __get_dynamic_len(dest)); \
992 const char *__ctf_tmp_string = \
993 ((_src) ? (_src) : __LTTNG_NULL_STRING); \
994 lib_ring_buffer_align_ctx(&__ctx, \
995 lttng_alignof(*__ctf_tmp_string)); \
996 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string, \
997 __get_dynamic_len(dest)); \
1001 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite) \
1002 _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
1005 #define ctf_align(_type) \
1006 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
1008 #undef ctf_custom_field
1009 #define ctf_custom_field(_type, _item, _code) _code
1011 #undef ctf_custom_code
1012 #define ctf_custom_code(...) \
1017 /* Beware: this get len actually consumes the len value */
1018 #undef __get_dynamic_len
1019 #define __get_dynamic_len(field) this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx++]
1022 #define TP_PROTO(...) __VA_ARGS__
1025 #define TP_ARGS(...) __VA_ARGS__
1028 #define TP_FIELDS(...) __VA_ARGS__
1031 #define TP_locvar(...) __VA_ARGS__
1034 #define TP_code_pre(...) __VA_ARGS__
1037 #define TP_code_post(...) __VA_ARGS__
1040 * For state dump, check that "session" argument (mandatory) matches the
1041 * session this event belongs to. Ensures that we write state dump data only
1042 * into the started session, not into all sessions.
1044 #ifdef TP_SESSION_CHECK
1045 #define _TP_SESSION_CHECK(session, csession) (session == csession)
1046 #else /* TP_SESSION_CHECK */
1047 #define _TP_SESSION_CHECK(session, csession) 1
1048 #endif /* TP_SESSION_CHECK */
1051 * Using twice size for filter stack data to hold size and pointer for
1052 * each field (worse case). For integers, max size required is 64-bit.
1053 * Same for double-precision floats. Those fit within
1054 * 2*sizeof(unsigned long) for all supported architectures.
1055 * Perform UNION (||) of filter runtime list.
1057 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1058 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1059 static void __event_probe__##_name(void *__data, _proto) \
1061 struct probe_local_vars { _locvar }; \
1062 struct lttng_event *__event = __data; \
1063 struct lttng_probe_ctx __lttng_probe_ctx = { \
1065 .event_notifier = NULL, \
1066 .interruptible = !irqs_disabled(), \
1068 struct lttng_channel *__chan = __event->chan; \
1069 struct lttng_session *__session = __chan->session; \
1070 struct lib_ring_buffer_ctx __ctx; \
1071 ssize_t __event_len; \
1072 size_t __event_align; \
1073 size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1075 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \
1076 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1079 struct probe_local_vars __tp_locvar; \
1080 struct probe_local_vars *tp_locvar __attribute__((unused)) = \
1082 struct lttng_id_tracker_rcu *__lf; \
1084 if (!_TP_SESSION_CHECK(session, __session)) \
1086 if (unlikely(!LTTNG_READ_ONCE(__session->active))) \
1088 if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \
1090 if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \
1092 __lf = lttng_rcu_dereference(__session->pid_tracker.p); \
1093 if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \
1095 __lf = lttng_rcu_dereference(__session->vpid_tracker.p); \
1096 if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1098 __lf = lttng_rcu_dereference(__session->uid_tracker.p); \
1099 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1100 lttng_current_uid()))) \
1102 __lf = lttng_rcu_dereference(__session->vuid_tracker.p); \
1103 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1104 lttng_current_vuid()))) \
1106 __lf = lttng_rcu_dereference(__session->gid_tracker.p); \
1107 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1108 lttng_current_gid()))) \
1110 __lf = lttng_rcu_dereference(__session->vgid_tracker.p); \
1111 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1112 lttng_current_vgid()))) \
1114 __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \
1115 __dynamic_len_idx = __orig_dynamic_len_offset; \
1117 if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \
1118 struct lttng_bytecode_runtime *bc_runtime; \
1119 int __filter_record = __event->has_enablers_without_bytecode; \
1121 __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \
1122 tp_locvar, _args); \
1123 lttng_list_for_each_entry_rcu(bc_runtime, &__event->filter_bytecode_runtime_head, node) { \
1124 if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \
1125 __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \
1126 __filter_record = 1; \
1130 if (likely(!__filter_record)) \
1133 __event_len = __event_get_size__##_name(tp_locvar, _args); \
1134 if (unlikely(__event_len < 0)) { \
1135 lib_ring_buffer_lost_event_too_big(__chan->chan); \
1138 __event_align = __event_get_align__##_name(tp_locvar, _args); \
1139 lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \
1140 __event_align, -1); \
1141 __ret = __chan->ops->event_reserve(&__ctx, __event->id); \
1145 __chan->ops->event_commit(&__ctx); \
1148 barrier(); /* use before un-reserve. */ \
1149 this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1153 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1154 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1155 static void __event_probe__##_name(void *__data) \
1157 struct probe_local_vars { _locvar }; \
1158 struct lttng_event *__event = __data; \
1159 struct lttng_probe_ctx __lttng_probe_ctx = { \
1161 .event_notifier = NULL, \
1162 .interruptible = !irqs_disabled(), \
1164 struct lttng_channel *__chan = __event->chan; \
1165 struct lttng_session *__session = __chan->session; \
1166 struct lib_ring_buffer_ctx __ctx; \
1167 ssize_t __event_len; \
1168 size_t __event_align; \
1169 size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1171 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \
1172 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1175 struct probe_local_vars __tp_locvar; \
1176 struct probe_local_vars *tp_locvar __attribute__((unused)) = \
1178 struct lttng_id_tracker_rcu *__lf; \
1180 if (!_TP_SESSION_CHECK(session, __session)) \
1182 if (unlikely(!LTTNG_READ_ONCE(__session->active))) \
1184 if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \
1186 if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \
1188 __lf = lttng_rcu_dereference(__session->pid_tracker.p); \
1189 if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \
1191 __lf = lttng_rcu_dereference(__session->vpid_tracker.p); \
1192 if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1194 __lf = lttng_rcu_dereference(__session->uid_tracker.p); \
1195 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1196 lttng_current_uid()))) \
1198 __lf = lttng_rcu_dereference(__session->vuid_tracker.p); \
1199 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1200 lttng_current_vuid()))) \
1202 __lf = lttng_rcu_dereference(__session->gid_tracker.p); \
1203 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1204 lttng_current_gid()))) \
1206 __lf = lttng_rcu_dereference(__session->vgid_tracker.p); \
1207 if (__lf && likely(!lttng_id_tracker_lookup(__lf, \
1208 lttng_current_vgid()))) \
1210 __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \
1211 __dynamic_len_idx = __orig_dynamic_len_offset; \
1213 if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \
1214 struct lttng_bytecode_runtime *bc_runtime; \
1215 int __filter_record = __event->has_enablers_without_bytecode; \
1217 __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \
1219 lttng_list_for_each_entry_rcu(bc_runtime, &__event->filter_bytecode_runtime_head, node) { \
1220 if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \
1221 __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) { \
1222 __filter_record = 1; \
1226 if (likely(!__filter_record)) \
1229 __event_len = __event_get_size__##_name(tp_locvar); \
1230 if (unlikely(__event_len < 0)) { \
1231 lib_ring_buffer_lost_event_too_big(__chan->chan); \
1234 __event_align = __event_get_align__##_name(tp_locvar); \
1235 lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \
1236 __event_align, -1); \
1237 __ret = __chan->ops->event_reserve(&__ctx, __event->id); \
1241 __chan->ops->event_commit(&__ctx); \
1244 barrier(); /* use before un-reserve. */ \
1245 this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1249 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1251 #undef __get_dynamic_len
1254 * Stage 6.1 of tracepoint generation: generate event notifier probes
1256 * Create the probe function. This function evaluates the filter bytecode and
1257 * queue a notification to be sent to userspace.
1260 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1263 #define TP_PROTO(...) __VA_ARGS__
1266 #define TP_ARGS(...) __VA_ARGS__
1269 #define TP_FIELDS(...) __VA_ARGS__
1272 #define TP_locvar(...) __VA_ARGS__
1275 #define TP_code_pre(...) __VA_ARGS__
1278 #define TP_code_post(...) __VA_ARGS__
1281 * Using twice size for filter stack data to hold size and pointer for
1282 * each field (worse case). For integers, max size required is 64-bit.
1283 * Same for double-precision floats. Those fit within
1284 * 2*sizeof(unsigned long) for all supported architectures.
1285 * Perform UNION (||) of filter runtime list.
1287 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1288 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1289 static void __event_notifier_probe__##_name(void *__data, _proto) \
1291 struct probe_local_vars { _locvar }; \
1292 struct lttng_event_notifier *__event_notifier = __data; \
1293 struct lttng_probe_ctx __lttng_probe_ctx = { \
1295 .event_notifier = __event_notifier, \
1296 .interruptible = !irqs_disabled(), \
1299 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \
1300 char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1302 struct probe_local_vars __tp_locvar; \
1303 struct probe_local_vars *tp_locvar __attribute__((unused)) = \
1305 struct lttng_kernel_notifier_ctx __notif_ctx; \
1306 bool __interpreter_stack_prepared = false; \
1308 if (unlikely(!READ_ONCE(__event_notifier->enabled))) \
1311 if (unlikely(!list_empty(&__event_notifier->filter_bytecode_runtime_head))) { \
1312 struct lttng_bytecode_runtime *bc_runtime; \
1313 int __filter_record = __event_notifier->has_enablers_without_bytecode; \
1315 __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \
1316 tp_locvar, _args); \
1317 __interpreter_stack_prepared = true; \
1318 lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \
1319 if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \
1320 __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \
1321 __filter_record = 1; \
1323 if (likely(!__filter_record)) \
1327 __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \
1328 if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \
1329 __event_prepare_interpreter_stack__##_name( \
1330 __stackvar.__interpreter_stack_data, \
1331 tp_locvar, _args); \
1333 __event_notifier->send_notification(__event_notifier, \
1334 &__lttng_probe_ctx, \
1335 __stackvar.__interpreter_stack_data, \
1343 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1344 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1345 static void __event_notifier_probe__##_name(void *__data) \
1347 struct probe_local_vars { _locvar }; \
1348 struct lttng_event_notifier *__event_notifier = __data; \
1349 struct lttng_probe_ctx __lttng_probe_ctx = { \
1351 .event_notifier = __event_notifier, \
1352 .interruptible = !irqs_disabled(), \
1355 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \
1356 char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1358 struct probe_local_vars __tp_locvar; \
1359 struct probe_local_vars *tp_locvar __attribute__((unused)) = \
1361 struct lttng_kernel_notifier_ctx __notif_ctx; \
1362 bool __interpreter_stack_prepared = false; \
1364 if (unlikely(!READ_ONCE(__event_notifier->enabled))) \
1367 if (unlikely(!list_empty(&__event_notifier->filter_bytecode_runtime_head))) { \
1368 struct lttng_bytecode_runtime *bc_runtime; \
1369 int __filter_record = __event_notifier->has_enablers_without_bytecode; \
1371 __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \
1373 __interpreter_stack_prepared = true; \
1374 lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \
1375 if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \
1376 __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \
1377 __filter_record = 1; \
1379 if (likely(!__filter_record)) \
1383 __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \
1384 if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \
1385 __event_prepare_interpreter_stack__##_name( \
1386 __stackvar.__interpreter_stack_data, \
1389 __event_notifier->send_notification(__event_notifier, \
1390 &__lttng_probe_ctx, \
1391 __stackvar.__interpreter_stack_data, \
1398 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1400 * Stage 7 of the trace events.
1402 * Create event descriptions.
1405 /* Named field types must be defined in lttng-types.h */
1407 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1410 #define TP_PROBE_CB(_template) &__event_probe__##_template
1413 #ifndef TP_EVENT_NOTIFIER_PROBE_CB
1414 #define TP_EVENT_NOTIFIER_PROBE_CB(_template) &__event_notifier_probe__##_template
1417 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1418 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1419 static const struct lttng_kernel_event_desc __event_desc___##_map = { \
1420 .event_name = #_map, \
1421 .event_kname = #_name, \
1422 .probe_callback = (void *) TP_PROBE_CB(_template), \
1423 .fields = __event_fields___##_template, \
1424 .nr_fields = ARRAY_SIZE(__event_fields___##_template), \
1425 .owner = THIS_MODULE, \
1426 .event_notifier_callback = (void *) TP_EVENT_NOTIFIER_PROBE_CB(_template), \
1429 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1430 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1431 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1433 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1436 * Stage 8 of the trace events.
1438 * Create an array of event description pointers.
1441 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1443 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1444 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1445 &__event_desc___##_map,
1447 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1448 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1449 LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1451 #define TP_ID1(_token, _system) _token##_system
1452 #define TP_ID(_token, _system) TP_ID1(_token, _system)
1454 static const struct lttng_kernel_event_desc
*TP_ID(__event_desc___
, TRACE_SYSTEM
)[] = {
1455 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1462 * Stage 9 of the trace events.
1464 * Create a toplevel descriptor for the whole probe.
1467 #define TP_ID1(_token, _system) _token##_system
1468 #define TP_ID(_token, _system) TP_ID1(_token, _system)
1470 /* non-const because list head will be modified when registered. */
1471 static __used
struct lttng_kernel_probe_desc
TP_ID(__probe_desc___
, TRACE_SYSTEM
) = {
1472 .provider_name
= __stringify(TRACE_SYSTEM
),
1473 .event_desc
= TP_ID(__event_desc___
, TRACE_SYSTEM
),
1474 .nr_events
= ARRAY_SIZE(TP_ID(__event_desc___
, TRACE_SYSTEM
)),
1475 .head
= { NULL
, NULL
},
1476 .lazy_init_head
= { NULL
, NULL
},
1484 * Stage 10 of the trace events.
1486 * Register/unregister probes at module load/unload.
1489 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1491 #define TP_ID1(_token, _system) _token##_system
1492 #define TP_ID(_token, _system) TP_ID1(_token, _system)
1493 #define module_init_eval1(_token, _system) module_init(_token##_system)
1494 #define module_init_eval(_token, _system) module_init_eval1(_token, _system)
1495 #define module_exit_eval1(_token, _system) module_exit(_token##_system)
1496 #define module_exit_eval(_token, _system) module_exit_eval1(_token, _system)
1498 #ifndef TP_MODULE_NOINIT
1499 static int TP_ID(__lttng_events_init__
, TRACE_SYSTEM
)(void)
1501 wrapper_vmalloc_sync_mappings();
1502 return lttng_probe_register(&TP_ID(__probe_desc___
, TRACE_SYSTEM
));
1505 static void TP_ID(__lttng_events_exit__
, TRACE_SYSTEM
)(void)
1507 lttng_probe_unregister(&TP_ID(__probe_desc___
, TRACE_SYSTEM
));
1510 #ifndef TP_MODULE_NOAUTOLOAD
1511 module_init_eval(__lttng_events_init__
, TRACE_SYSTEM
);
1512 module_exit_eval(__lttng_events_exit__
, TRACE_SYSTEM
);
1517 #undef module_init_eval
1518 #undef module_exit_eval
This page took 0.132372 seconds and 4 git commands to generate.