Commit | Line | Data |
---|---|---|
21f58fb7 FD |
1 | /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) |
2 | * | |
3 | * lttng-event-notifier-notification.c | |
4 | * | |
5 | * Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com> | |
6 | */ | |
7 | ||
42f3ef8c FD |
8 | #include <linux/bug.h> |
9 | ||
10 | #include <lttng/lttng-bytecode.h> | |
21f58fb7 | 11 | #include <lttng/events.h> |
42f3ef8c | 12 | #include <lttng/msgpack.h> |
21f58fb7 | 13 | #include <lttng/event-notifier-notification.h> |
437d5aa5 | 14 | #include <lttng/events-internal.h> |
18789896 | 15 | #include <lttng/probe-user.h> |
6657edec | 16 | #include <wrapper/barrier.h> |
21f58fb7 | 17 | |
99d223ad | 18 | /* |
e699ee6c MD |
19 | * The capture buffer size needs to be below 1024 bytes to avoid the |
20 | * frame to be larger than the 1024 limit enforced by the kernel. If we | |
21 | * ever need to increase it, we will need to use a memory allocation | |
22 | * scheme which allows allocating temporary memory chunks from the | |
23 | * instrumentation sites. This could be done by adapting lttng | |
24 | * tp-mempool to become nmi-safe and lock-free. | |
99d223ad FD |
25 | */ |
26 | #define CAPTURE_BUFFER_SIZE 512 | |
27 | ||
b9315ad8 MD |
28 | #define MSG_WRITE_NIL_LEN 1 |
29 | ||
99d223ad FD |
30 | struct lttng_event_notifier_notification { |
31 | int notification_fd; | |
32 | uint64_t event_notifier_token; | |
33 | uint8_t capture_buf[CAPTURE_BUFFER_SIZE]; | |
34 | struct lttng_msgpack_writer writer; | |
35 | bool has_captures; | |
36 | }; | |
37 | ||
42f3ef8c FD |
38 | static |
39 | int capture_enum(struct lttng_msgpack_writer *writer, | |
40 | struct lttng_interpreter_output *output) | |
41 | { | |
42 | int ret; | |
43 | ||
44 | /* | |
45 | * Enums are captured as a map containing 2 key-value pairs. Such as: | |
46 | * - type: enum | |
47 | * value: 177 | |
48 | */ | |
49 | ret = lttng_msgpack_begin_map(writer, 2); | |
50 | if (ret) { | |
42f3ef8c FD |
51 | goto end; |
52 | } | |
53 | ||
54 | ret = lttng_msgpack_write_str(writer, "type"); | |
55 | if (ret) { | |
42f3ef8c FD |
56 | goto end; |
57 | } | |
58 | ||
59 | ret = lttng_msgpack_write_str(writer, "enum"); | |
60 | if (ret) { | |
42f3ef8c FD |
61 | goto end; |
62 | } | |
63 | ||
64 | ret = lttng_msgpack_write_str(writer, "value"); | |
65 | if (ret) { | |
42f3ef8c FD |
66 | goto end; |
67 | } | |
68 | ||
69 | switch (output->type) { | |
70 | case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM: | |
71 | ret = lttng_msgpack_write_signed_integer(writer, output->u.s); | |
72 | if (ret) { | |
42f3ef8c FD |
73 | goto end; |
74 | } | |
75 | break; | |
76 | case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM: | |
77 | ret = lttng_msgpack_write_signed_integer(writer, output->u.u); | |
78 | if (ret) { | |
42f3ef8c FD |
79 | goto end; |
80 | } | |
81 | break; | |
82 | default: | |
76db4808 | 83 | WARN_ON_ONCE(1); |
42f3ef8c FD |
84 | } |
85 | ||
86 | ret = lttng_msgpack_end_map(writer); | |
42f3ef8c FD |
87 | end: |
88 | return ret; | |
89 | } | |
90 | ||
91 | static | |
92 | int64_t capture_sequence_element_signed(uint8_t *ptr, | |
437d5aa5 | 93 | const struct lttng_kernel_type_integer *type) |
42f3ef8c FD |
94 | { |
95 | int64_t value = 0; | |
96 | unsigned int size = type->size; | |
18789896 | 97 | bool user = type->user; |
42f3ef8c FD |
98 | bool byte_order_reversed = type->reverse_byte_order; |
99 | ||
100 | switch (size) { | |
101 | case 8: | |
18789896 MD |
102 | { |
103 | int8_t tmp; | |
104 | ||
105 | if (user) { | |
106 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int8_t))) | |
107 | tmp = 0; | |
108 | } else { | |
109 | tmp = *ptr; | |
110 | } | |
111 | value = tmp; | |
42f3ef8c | 112 | break; |
18789896 | 113 | } |
42f3ef8c FD |
114 | case 16: |
115 | { | |
116 | int16_t tmp; | |
18789896 MD |
117 | |
118 | if (user) { | |
119 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int16_t))) | |
120 | tmp = 0; | |
121 | } else { | |
122 | tmp = *(int16_t *) ptr; | |
123 | } | |
42f3ef8c FD |
124 | if (byte_order_reversed) |
125 | __swab16s(&tmp); | |
42f3ef8c FD |
126 | value = tmp; |
127 | break; | |
128 | } | |
129 | case 32: | |
130 | { | |
131 | int32_t tmp; | |
18789896 MD |
132 | |
133 | if (user) { | |
134 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int32_t))) | |
135 | tmp = 0; | |
136 | } else { | |
137 | tmp = *(int32_t *) ptr; | |
138 | } | |
42f3ef8c FD |
139 | if (byte_order_reversed) |
140 | __swab32s(&tmp); | |
42f3ef8c FD |
141 | value = tmp; |
142 | break; | |
143 | } | |
144 | case 64: | |
145 | { | |
146 | int64_t tmp; | |
18789896 MD |
147 | |
148 | if (user) { | |
149 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int64_t))) | |
150 | tmp = 0; | |
151 | } else { | |
152 | tmp = *(int64_t *) ptr; | |
153 | } | |
42f3ef8c FD |
154 | if (byte_order_reversed) |
155 | __swab64s(&tmp); | |
42f3ef8c FD |
156 | value = tmp; |
157 | break; | |
158 | } | |
159 | default: | |
76db4808 | 160 | WARN_ON_ONCE(1); |
42f3ef8c FD |
161 | } |
162 | ||
163 | return value; | |
164 | } | |
165 | ||
166 | static | |
167 | uint64_t capture_sequence_element_unsigned(uint8_t *ptr, | |
437d5aa5 | 168 | const struct lttng_kernel_type_integer *type) |
42f3ef8c FD |
169 | { |
170 | uint64_t value = 0; | |
171 | unsigned int size = type->size; | |
18789896 | 172 | bool user = type->user; |
42f3ef8c FD |
173 | bool byte_order_reversed = type->reverse_byte_order; |
174 | ||
175 | switch (size) { | |
176 | case 8: | |
18789896 MD |
177 | { |
178 | uint8_t tmp; | |
179 | ||
180 | if (user) { | |
181 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint8_t))) | |
182 | tmp = 0; | |
183 | } else { | |
184 | tmp = *ptr; | |
185 | } | |
186 | value = tmp; | |
42f3ef8c | 187 | break; |
18789896 | 188 | } |
42f3ef8c FD |
189 | case 16: |
190 | { | |
191 | uint16_t tmp; | |
18789896 MD |
192 | |
193 | if (user) { | |
194 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint16_t))) | |
195 | tmp = 0; | |
196 | } else { | |
197 | tmp = *(uint16_t *) ptr; | |
198 | } | |
42f3ef8c FD |
199 | if (byte_order_reversed) |
200 | __swab16s(&tmp); | |
42f3ef8c FD |
201 | value = tmp; |
202 | break; | |
203 | } | |
204 | case 32: | |
205 | { | |
206 | uint32_t tmp; | |
18789896 MD |
207 | |
208 | if (user) { | |
209 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint32_t))) | |
210 | tmp = 0; | |
211 | } else { | |
212 | tmp = *(uint32_t *) ptr; | |
213 | } | |
42f3ef8c FD |
214 | if (byte_order_reversed) |
215 | __swab32s(&tmp); | |
42f3ef8c FD |
216 | value = tmp; |
217 | break; | |
218 | } | |
219 | case 64: | |
220 | { | |
221 | uint64_t tmp; | |
18789896 MD |
222 | |
223 | if (user) { | |
224 | if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint64_t))) | |
225 | tmp = 0; | |
226 | } else { | |
227 | tmp = *(uint64_t *) ptr; | |
228 | } | |
42f3ef8c FD |
229 | if (byte_order_reversed) |
230 | __swab64s(&tmp); | |
42f3ef8c FD |
231 | value = tmp; |
232 | break; | |
233 | } | |
234 | default: | |
76db4808 | 235 | WARN_ON_ONCE(1); |
42f3ef8c FD |
236 | } |
237 | ||
238 | return value; | |
239 | } | |
240 | ||
42f3ef8c FD |
241 | int capture_sequence(struct lttng_msgpack_writer *writer, |
242 | struct lttng_interpreter_output *output) | |
243 | { | |
437d5aa5 MD |
244 | const struct lttng_kernel_type_integer *integer_type = NULL; |
245 | const struct lttng_kernel_type_common *nested_type; | |
42f3ef8c FD |
246 | uint8_t *ptr; |
247 | bool signedness; | |
248 | int ret, i; | |
249 | ||
250 | ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem); | |
251 | if (ret) { | |
42f3ef8c FD |
252 | goto end; |
253 | } | |
254 | ||
255 | ptr = (uint8_t *) output->u.sequence.ptr; | |
256 | nested_type = output->u.sequence.nested_type; | |
12bb2edb MD |
257 | switch (nested_type->type) { |
258 | case lttng_kernel_type_integer: | |
437d5aa5 | 259 | integer_type = lttng_kernel_get_type_integer(nested_type); |
42f3ef8c | 260 | break; |
437d5aa5 | 261 | case lttng_kernel_type_enum: |
42f3ef8c | 262 | /* Treat enumeration as an integer. */ |
437d5aa5 | 263 | integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type); |
42f3ef8c FD |
264 | break; |
265 | default: | |
266 | /* Capture of array of non-integer are not supported. */ | |
76db4808 MD |
267 | WARN_ON_ONCE(1); |
268 | ret = -1; | |
269 | goto end; | |
42f3ef8c FD |
270 | } |
271 | signedness = integer_type->signedness; | |
272 | for (i = 0; i < output->u.sequence.nr_elem; i++) { | |
273 | if (signedness) { | |
274 | ret = lttng_msgpack_write_signed_integer(writer, | |
275 | capture_sequence_element_signed(ptr, integer_type)); | |
276 | if (ret) { | |
42f3ef8c FD |
277 | goto end; |
278 | } | |
279 | } else { | |
280 | ret = lttng_msgpack_write_unsigned_integer(writer, | |
281 | capture_sequence_element_unsigned(ptr, integer_type)); | |
282 | if (ret) { | |
42f3ef8c FD |
283 | goto end; |
284 | } | |
285 | } | |
286 | ||
287 | /* | |
288 | * We assume that alignment is smaller or equal to the size. | |
289 | * This currently holds true but if it changes in the future, | |
290 | * we will want to change the pointer arithmetics below to | |
291 | * take into account that the next element might be further | |
292 | * away. | |
293 | */ | |
76db4808 | 294 | WARN_ON_ONCE(integer_type->alignment > integer_type->size); |
42f3ef8c FD |
295 | |
296 | /* Size is in number of bits. */ | |
297 | ptr += (integer_type->size / CHAR_BIT) ; | |
298 | } | |
299 | ||
300 | ret = lttng_msgpack_end_array(writer); | |
42f3ef8c FD |
301 | end: |
302 | return ret; | |
303 | } | |
304 | ||
99d223ad FD |
305 | static |
306 | int notification_append_capture( | |
307 | struct lttng_event_notifier_notification *notif, | |
308 | struct lttng_interpreter_output *output) | |
309 | { | |
310 | struct lttng_msgpack_writer *writer = ¬if->writer; | |
311 | int ret = 0; | |
312 | ||
313 | switch (output->type) { | |
314 | case LTTNG_INTERPRETER_TYPE_S64: | |
315 | ret = lttng_msgpack_write_signed_integer(writer, output->u.s); | |
99d223ad FD |
316 | break; |
317 | case LTTNG_INTERPRETER_TYPE_U64: | |
318 | ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u); | |
99d223ad FD |
319 | break; |
320 | case LTTNG_INTERPRETER_TYPE_STRING: | |
9c16adb2 MD |
321 | if (output->u.str.user) { |
322 | ret = lttng_msgpack_write_user_str(writer, output->u.str.user_str); | |
323 | } else { | |
324 | ret = lttng_msgpack_write_str(writer, output->u.str.str); | |
325 | } | |
99d223ad FD |
326 | break; |
327 | case LTTNG_INTERPRETER_TYPE_SEQUENCE: | |
328 | ret = capture_sequence(writer, output); | |
99d223ad FD |
329 | break; |
330 | case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM: | |
331 | case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM: | |
332 | ret = capture_enum(writer, output); | |
99d223ad FD |
333 | break; |
334 | default: | |
335 | ret = -1; | |
76db4808 | 336 | WARN_ON_ONCE(1); |
99d223ad | 337 | } |
99d223ad FD |
338 | return ret; |
339 | } | |
340 | ||
341 | static | |
342 | int notification_append_empty_capture( | |
343 | struct lttng_event_notifier_notification *notif) | |
344 | { | |
76db4808 | 345 | return lttng_msgpack_write_nil(¬if->writer); |
99d223ad FD |
346 | } |
347 | ||
348 | static | |
349 | int notification_init(struct lttng_event_notifier_notification *notif, | |
a67ba386 | 350 | struct lttng_kernel_event_notifier *event_notifier) |
99d223ad FD |
351 | { |
352 | struct lttng_msgpack_writer *writer = ¬if->writer; | |
353 | int ret = 0; | |
354 | ||
355 | notif->has_captures = false; | |
356 | ||
a67ba386 | 357 | if (event_notifier->priv->num_captures > 0) { |
99d223ad FD |
358 | lttng_msgpack_writer_init(writer, notif->capture_buf, |
359 | CAPTURE_BUFFER_SIZE); | |
360 | ||
a67ba386 | 361 | ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures); |
99d223ad | 362 | if (ret) { |
99d223ad FD |
363 | goto end; |
364 | } | |
365 | ||
366 | notif->has_captures = true; | |
367 | } | |
368 | ||
369 | end: | |
370 | return ret; | |
371 | } | |
372 | ||
99f52fcc | 373 | static |
a67ba386 | 374 | void record_error(struct lttng_kernel_event_notifier *event_notifier) |
99f52fcc FD |
375 | { |
376 | ||
a67ba386 | 377 | struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group; |
ab04d7b1 | 378 | struct lttng_counter *error_counter; |
99f52fcc FD |
379 | size_t dimension_index[1]; |
380 | int ret; | |
381 | ||
ab04d7b1 | 382 | /* |
6657edec | 383 | * lttng_smp_load_acquire paired with lttng_smp_store_release orders |
ab04d7b1 MD |
384 | * creation of the error counter and setting error_counter_len |
385 | * before the error_counter is used. | |
386 | */ | |
6657edec | 387 | error_counter = lttng_smp_load_acquire(&event_notifier_group->error_counter); |
a83d6831 | 388 | /* This group may not have an error counter attached to it. */ |
ab04d7b1 | 389 | if (!error_counter) |
a83d6831 FD |
390 | return; |
391 | ||
a67ba386 | 392 | dimension_index[0] = event_notifier->priv->error_counter_index; |
99f52fcc | 393 | |
ab04d7b1 | 394 | ret = error_counter->ops->counter_add(error_counter->counter, |
99f52fcc FD |
395 | dimension_index, 1); |
396 | if (ret) | |
397 | WARN_ON_ONCE(1); | |
398 | } | |
399 | ||
99d223ad FD |
400 | static |
401 | void notification_send(struct lttng_event_notifier_notification *notif, | |
a67ba386 | 402 | struct lttng_kernel_event_notifier *event_notifier) |
21f58fb7 | 403 | { |
a67ba386 | 404 | struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group; |
e27e4381 | 405 | struct lttng_kernel_ring_buffer_ctx ctx; |
606828e4 | 406 | struct lttng_kernel_abi_event_notifier_notification kernel_notif; |
99d223ad | 407 | size_t capture_buffer_content_len, reserve_size; |
21f58fb7 FD |
408 | int ret; |
409 | ||
99d223ad | 410 | reserve_size = sizeof(kernel_notif); |
a67ba386 | 411 | kernel_notif.token = event_notifier->priv->parent.user_token; |
21f58fb7 | 412 | |
99d223ad FD |
413 | if (notif->has_captures) { |
414 | capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer; | |
415 | } else { | |
416 | capture_buffer_content_len = 0; | |
417 | } | |
418 | ||
419 | WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE); | |
420 | ||
421 | reserve_size += capture_buffer_content_len; | |
422 | kernel_notif.capture_buf_size = capture_buffer_content_len; | |
423 | ||
8a445457 | 424 | lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size, |
c7d9db61 | 425 | lttng_alignof(kernel_notif), NULL); |
cefd899f | 426 | ret = event_notifier_group->ops->event_reserve(&ctx); |
21f58fb7 | 427 | if (ret < 0) { |
99f52fcc | 428 | record_error(event_notifier); |
21f58fb7 FD |
429 | return; |
430 | } | |
99d223ad | 431 | |
99d223ad FD |
432 | /* Write the notif structure. */ |
433 | event_notifier_group->ops->event_write(&ctx, &kernel_notif, | |
77fcffef | 434 | sizeof(kernel_notif), lttng_alignof(kernel_notif)); |
99d223ad FD |
435 | |
436 | /* | |
437 | * Write the capture buffer. No need to realigned as the below is a raw | |
438 | * char* buffer. | |
439 | */ | |
440 | event_notifier_group->ops->event_write(&ctx, ¬if->capture_buf, | |
77fcffef | 441 | capture_buffer_content_len, 1); |
99d223ad | 442 | |
21f58fb7 FD |
443 | event_notifier_group->ops->event_commit(&ctx); |
444 | irq_work_queue(&event_notifier_group->wakeup_pending); | |
445 | } | |
99d223ad | 446 | |
b9315ad8 MD |
447 | /* |
448 | * Validate that the buffer has enough room to hold empty capture fields. | |
449 | */ | |
450 | static | |
451 | bool validate_buffer_len(struct lttng_event_notifier_notification *notif, size_t captures_left) | |
452 | { | |
453 | if (notif->writer.end_write_pos - notif->writer.write_pos < MSG_WRITE_NIL_LEN * captures_left) | |
454 | return false; | |
455 | return true; | |
456 | } | |
457 | ||
a67ba386 | 458 | void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier, |
c3eddb2e | 459 | const char *stack_data, |
346cb5ee | 460 | struct lttng_kernel_probe_ctx *probe_ctx, |
a67ba386 | 461 | struct lttng_kernel_notification_ctx *notif_ctx) |
99d223ad FD |
462 | { |
463 | struct lttng_event_notifier_notification notif = { 0 }; | |
b9315ad8 | 464 | size_t captures_left; |
99d223ad | 465 | |
b9315ad8 MD |
466 | if (notification_init(¬if, event_notifier)) |
467 | goto error; | |
468 | ||
469 | captures_left = event_notifier->priv->num_captures; | |
470 | if (!validate_buffer_len(¬if, captures_left)) | |
471 | goto error; | |
99d223ad | 472 | |
c3eddb2e | 473 | if (unlikely(notif_ctx->eval_capture)) { |
dc7350d6 | 474 | struct lttng_kernel_bytecode_runtime *capture_bc_runtime; |
99d223ad FD |
475 | |
476 | /* | |
477 | * Iterate over all the capture bytecodes. If the interpreter | |
478 | * functions returns successfully, append the value of the | |
479 | * `output` parameter to the capture buffer. If the interpreter | |
480 | * fails, append an empty capture to the buffer. | |
481 | */ | |
c3eddb2e | 482 | list_for_each_entry_rcu(capture_bc_runtime, |
a67ba386 | 483 | &event_notifier->priv->capture_bytecode_runtime_head, node) { |
99d223ad | 484 | struct lttng_interpreter_output output; |
76db4808 | 485 | uint8_t *save_pos; |
b9315ad8 | 486 | int ret = -1; |
99d223ad | 487 | |
76db4808 | 488 | lttng_msgpack_save_writer_pos(¬if.writer, &save_pos); |
b9315ad8 | 489 | captures_left--; |
8a445457 MD |
490 | if (capture_bc_runtime->interpreter_func(capture_bc_runtime, |
491 | stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK) | |
99d223ad | 492 | ret = notification_append_capture(¬if, &output); |
b9315ad8 | 493 | if (ret || !validate_buffer_len(¬if, captures_left)) { |
76db4808 | 494 | /* |
b9315ad8 MD |
495 | * On append capture error or if the generated |
496 | * buffer data would not leave enough room to | |
497 | * write empty capture fields for the remaining | |
498 | * fields, skip the field capture by restoring | |
499 | * the msgpack writer position and writing an | |
500 | * empty capture field. | |
76db4808 MD |
501 | */ |
502 | lttng_msgpack_restore_writer_pos(¬if.writer, save_pos); | |
b9315ad8 MD |
503 | ret = notification_append_empty_capture(¬if); |
504 | WARN_ON_ONCE(ret); | |
76db4808 | 505 | } |
99d223ad FD |
506 | } |
507 | } | |
508 | ||
b9315ad8 MD |
509 | if (notif.has_captures && lttng_msgpack_end_array(¬if.writer)) |
510 | goto error; | |
511 | ||
99d223ad FD |
512 | /* |
513 | * Send the notification (including the capture buffer) to the | |
514 | * sessiond. | |
515 | */ | |
516 | notification_send(¬if, event_notifier); | |
b9315ad8 MD |
517 | return; |
518 | ||
519 | error: | |
520 | record_error(event_notifier); | |
99d223ad FD |
521 | return; |
522 | } |