2 * SPDX-License-Identifier: LGPL-2.1-only
4 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * LTTng UST performance monitoring counters (perf-counters) integration.
10 #include <sys/types.h>
19 #include <sys/syscall.h>
20 #include <lttng/ust-arch.h>
21 #include <lttng/ust-events.h>
22 #include <lttng/ust-tracer.h>
23 #include <lttng/ringbuffer-context.h>
24 #include <urcu/system.h>
25 #include <urcu/arch.h>
26 #include <urcu/rculist.h>
27 #include <ust-helper.h>
29 #include <usterr-signal-safe.h>
31 #include <urcu/tls-compat.h>
32 #include "perf_event.h"
34 #include "context-internal.h"
35 #include "lttng-tracer-core.h"
36 #include "ust-events-internal.h"
39 * We use a global perf counter key and iterate on per-thread RCU lists
40 * of fields in the fast path, even though this is not strictly speaking
41 * what would provide the best fast-path complexity, to ensure teardown
42 * of sessions vs thread exit is handled racelessly.
44 * Updates and traversals of thread_list are protected by UST lock.
45 * Updates to rcu_field_list are protected by UST lock.
48 struct lttng_perf_counter_thread_field
{
49 struct lttng_perf_counter_field
*field
; /* Back reference */
50 struct perf_event_mmap_page
*pc
;
51 struct cds_list_head thread_field_node
; /* Per-field list of thread fields (node) */
52 struct cds_list_head rcu_field_node
; /* RCU per-thread list of fields (node) */
56 struct lttng_perf_counter_thread
{
57 struct cds_list_head rcu_field_list
; /* RCU per-thread list of fields */
60 struct lttng_perf_counter_field
{
61 struct perf_event_attr attr
;
62 struct cds_list_head thread_field_list
; /* Per-field list of thread fields */
65 static pthread_key_t perf_counter_key
;
68 * lttng_perf_lock - Protect lttng-ust perf counter data structures
70 * Nests within the ust_lock, and therefore within the libc dl lock.
71 * Therefore, we need to fixup the TLS before nesting into this lock.
72 * Nests inside RCU bp read-side lock. Protects against concurrent
75 static pthread_mutex_t ust_perf_mutex
= PTHREAD_MUTEX_INITIALIZER
;
78 * Cancel state when grabbing the ust_perf_mutex. Saved when locking,
79 * restored on unlock. Protected by ust_perf_mutex.
81 static int ust_perf_saved_cancelstate
;
84 * Track whether we are tracing from a signal handler nested on an
87 static DEFINE_URCU_TLS(int, ust_perf_mutex_nest
);
90 * Force a read (imply TLS fixup for dlopen) of TLS variables.
92 void lttng_ust_fixup_perf_counter_tls(void)
94 asm volatile ("" : : "m" (URCU_TLS(ust_perf_mutex_nest
)));
97 void lttng_perf_lock(void)
99 sigset_t sig_all_blocked
, orig_mask
;
102 ret
= pthread_setcancelstate(PTHREAD_CANCEL_DISABLE
, &oldstate
);
104 ERR("pthread_setcancelstate: %s", strerror(ret
));
106 sigfillset(&sig_all_blocked
);
107 ret
= pthread_sigmask(SIG_SETMASK
, &sig_all_blocked
, &orig_mask
);
109 ERR("pthread_sigmask: %s", strerror(ret
));
111 if (!URCU_TLS(ust_perf_mutex_nest
)++) {
113 * Ensure the compiler don't move the store after the close()
114 * call in case close() would be marked as leaf.
117 pthread_mutex_lock(&ust_perf_mutex
);
118 ust_perf_saved_cancelstate
= oldstate
;
120 ret
= pthread_sigmask(SIG_SETMASK
, &orig_mask
, NULL
);
122 ERR("pthread_sigmask: %s", strerror(ret
));
126 void lttng_perf_unlock(void)
128 sigset_t sig_all_blocked
, orig_mask
;
129 int ret
, newstate
, oldstate
;
130 bool restore_cancel
= false;
132 sigfillset(&sig_all_blocked
);
133 ret
= pthread_sigmask(SIG_SETMASK
, &sig_all_blocked
, &orig_mask
);
135 ERR("pthread_sigmask: %s", strerror(ret
));
138 * Ensure the compiler don't move the store before the close()
139 * call, in case close() would be marked as leaf.
142 if (!--URCU_TLS(ust_perf_mutex_nest
)) {
143 newstate
= ust_perf_saved_cancelstate
;
144 restore_cancel
= true;
145 pthread_mutex_unlock(&ust_perf_mutex
);
147 ret
= pthread_sigmask(SIG_SETMASK
, &orig_mask
, NULL
);
149 ERR("pthread_sigmask: %s", strerror(ret
));
151 if (restore_cancel
) {
152 ret
= pthread_setcancelstate(newstate
, &oldstate
);
154 ERR("pthread_setcancelstate: %s", strerror(ret
));
160 size_t perf_counter_get_size(struct lttng_ust_ctx_field
*field
, size_t offset
)
164 size
+= lib_ring_buffer_align(offset
, lttng_alignof(uint64_t));
165 size
+= sizeof(uint64_t);
170 uint64_t read_perf_counter_syscall(
171 struct lttng_perf_counter_thread_field
*thread_field
)
175 if (caa_unlikely(thread_field
->fd
< 0))
178 if (caa_unlikely(read(thread_field
->fd
, &count
, sizeof(count
))
185 #if defined(LTTNG_UST_ARCH_X86)
188 uint64_t rdpmc(unsigned int counter
)
190 unsigned int low
, high
;
192 asm volatile("rdpmc" : "=a" (low
), "=d" (high
) : "c" (counter
));
194 return low
| ((uint64_t) high
) << 32;
198 bool has_rdpmc(struct perf_event_mmap_page
*pc
)
200 if (caa_unlikely(!pc
->cap_bit0_is_deprecated
))
202 /* Since Linux kernel 3.12. */
203 return pc
->cap_user_rdpmc
;
207 uint64_t arch_read_perf_counter(
208 struct lttng_perf_counter_thread_field
*thread_field
)
212 struct perf_event_mmap_page
*pc
= thread_field
->pc
;
214 if (caa_unlikely(!pc
))
218 seq
= CMM_LOAD_SHARED(pc
->lock
);
222 if (caa_likely(has_rdpmc(pc
) && idx
)) {
225 pmcval
= rdpmc(idx
- 1);
226 /* Sign-extend the pmc register result. */
227 pmcval
<<= 64 - pc
->pmc_width
;
228 pmcval
>>= 64 - pc
->pmc_width
;
229 count
= pc
->offset
+ pmcval
;
231 /* Fall-back on system call if rdpmc cannot be used. */
232 return read_perf_counter_syscall(thread_field
);
235 } while (CMM_LOAD_SHARED(pc
->lock
) != seq
);
241 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field
*thread_field
)
243 struct perf_event_mmap_page
*pc
= thread_field
->pc
;
247 return !has_rdpmc(pc
);
252 /* Generic (slow) implementation using a read system call. */
254 uint64_t arch_read_perf_counter(
255 struct lttng_perf_counter_thread_field
*thread_field
)
257 return read_perf_counter_syscall(thread_field
);
261 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field
*thread_field
)
269 int sys_perf_event_open(struct perf_event_attr
*attr
,
270 pid_t pid
, int cpu
, int group_fd
,
273 return syscall(SYS_perf_event_open
, attr
, pid
, cpu
,
278 int open_perf_fd(struct perf_event_attr
*attr
)
282 fd
= sys_perf_event_open(attr
, 0, -1, -1, 0);
290 void close_perf_fd(int fd
)
299 perror("Error closing LTTng-UST perf memory mapping FD");
303 static void setup_perf(struct lttng_perf_counter_thread_field
*thread_field
)
307 perf_addr
= mmap(NULL
, sizeof(struct perf_event_mmap_page
),
308 PROT_READ
, MAP_SHARED
, thread_field
->fd
, 0);
309 if (perf_addr
== MAP_FAILED
)
311 thread_field
->pc
= perf_addr
;
313 if (!arch_perf_keep_fd(thread_field
)) {
314 close_perf_fd(thread_field
->fd
);
315 thread_field
->fd
= -1;
320 void unmap_perf_page(struct perf_event_mmap_page
*pc
)
326 ret
= munmap(pc
, sizeof(struct perf_event_mmap_page
));
328 PERROR("Error in munmap");
334 struct lttng_perf_counter_thread
*alloc_perf_counter_thread(void)
336 struct lttng_perf_counter_thread
*perf_thread
;
337 sigset_t newmask
, oldmask
;
340 ret
= sigfillset(&newmask
);
343 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
346 /* Check again with signals disabled */
347 perf_thread
= pthread_getspecific(perf_counter_key
);
350 perf_thread
= zmalloc(sizeof(*perf_thread
));
353 CDS_INIT_LIST_HEAD(&perf_thread
->rcu_field_list
);
354 ret
= pthread_setspecific(perf_counter_key
, perf_thread
);
358 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
365 struct lttng_perf_counter_thread_field
*
366 add_thread_field(struct lttng_perf_counter_field
*perf_field
,
367 struct lttng_perf_counter_thread
*perf_thread
)
369 struct lttng_perf_counter_thread_field
*thread_field
;
370 sigset_t newmask
, oldmask
;
373 ret
= sigfillset(&newmask
);
376 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
379 /* Check again with signals disabled */
380 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
382 if (thread_field
->field
== perf_field
)
385 thread_field
= zmalloc(sizeof(*thread_field
));
388 thread_field
->field
= perf_field
;
389 thread_field
->fd
= open_perf_fd(&perf_field
->attr
);
390 if (thread_field
->fd
>= 0)
391 setup_perf(thread_field
);
393 * Note: thread_field->pc can be NULL if setup_perf() fails.
394 * Also, thread_field->fd can be -1 if open_perf_fd() fails.
397 cds_list_add_rcu(&thread_field
->rcu_field_node
,
398 &perf_thread
->rcu_field_list
);
399 cds_list_add(&thread_field
->thread_field_node
,
400 &perf_field
->thread_field_list
);
403 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
410 struct lttng_perf_counter_thread_field
*
411 get_thread_field(struct lttng_perf_counter_field
*field
)
413 struct lttng_perf_counter_thread
*perf_thread
;
414 struct lttng_perf_counter_thread_field
*thread_field
;
416 perf_thread
= pthread_getspecific(perf_counter_key
);
418 perf_thread
= alloc_perf_counter_thread();
419 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
421 if (thread_field
->field
== field
)
424 /* perf_counter_thread_field not found, need to add one */
425 return add_thread_field(field
, perf_thread
);
429 uint64_t wrapper_perf_counter_read(struct lttng_ust_ctx_field
*field
)
431 struct lttng_perf_counter_field
*perf_field
;
432 struct lttng_perf_counter_thread_field
*perf_thread_field
;
434 perf_field
= (struct lttng_perf_counter_field
*) field
->priv
;
435 perf_thread_field
= get_thread_field(perf_field
);
436 return arch_read_perf_counter(perf_thread_field
);
440 void perf_counter_record(struct lttng_ust_ctx_field
*field
,
441 struct lttng_ust_lib_ring_buffer_ctx
*ctx
,
442 struct lttng_ust_channel_buffer
*chan
)
446 value
= wrapper_perf_counter_read(field
);
447 lib_ring_buffer_align_ctx(ctx
, lttng_alignof(value
));
448 chan
->ops
->event_write(ctx
, &value
, sizeof(value
));
452 void perf_counter_get_value(struct lttng_ust_ctx_field
*field
,
453 struct lttng_ust_ctx_value
*value
)
455 value
->u
.s64
= wrapper_perf_counter_read(field
);
458 /* Called with perf lock held */
460 void lttng_destroy_perf_thread_field(
461 struct lttng_perf_counter_thread_field
*thread_field
)
463 close_perf_fd(thread_field
->fd
);
464 unmap_perf_page(thread_field
->pc
);
465 cds_list_del_rcu(&thread_field
->rcu_field_node
);
466 cds_list_del(&thread_field
->thread_field_node
);
471 void lttng_destroy_perf_thread_key(void *_key
)
473 struct lttng_perf_counter_thread
*perf_thread
= _key
;
474 struct lttng_perf_counter_thread_field
*pos
, *p
;
477 cds_list_for_each_entry_safe(pos
, p
, &perf_thread
->rcu_field_list
,
479 lttng_destroy_perf_thread_field(pos
);
484 /* Called with UST lock held */
486 void lttng_destroy_perf_counter_field(struct lttng_ust_ctx_field
*field
)
488 struct lttng_perf_counter_field
*perf_field
;
489 struct lttng_perf_counter_thread_field
*pos
, *p
;
491 free((char *) field
->event_field
->name
);
492 perf_field
= (struct lttng_perf_counter_field
*) field
->priv
;
494 * This put is performed when no threads can concurrently
495 * perform a "get" concurrently, thanks to urcu-bp grace
496 * period. Holding the lttng perf lock protects against
497 * concurrent modification of the per-thread thread field
501 cds_list_for_each_entry_safe(pos
, p
, &perf_field
->thread_field_list
,
503 lttng_destroy_perf_thread_field(pos
);
508 #ifdef LTTNG_UST_ARCH_ARMV7
511 int perf_get_exclude_kernel(void)
516 #else /* LTTNG_UST_ARCH_ARMV7 */
519 int perf_get_exclude_kernel(void)
524 #endif /* LTTNG_UST_ARCH_ARMV7 */
526 /* Called with UST lock held */
527 int lttng_add_perf_counter_to_ctx(uint32_t type
,
530 struct lttng_ust_ctx
**ctx
)
532 struct lttng_ust_ctx_field
*field
;
533 struct lttng_ust_type_common
*ust_type
;
534 struct lttng_perf_counter_field
*perf_field
;
538 name_alloc
= strdup(name
);
541 goto name_alloc_error
;
543 perf_field
= zmalloc(sizeof(*perf_field
));
546 goto perf_field_alloc_error
;
548 ust_type
= lttng_ust_create_type_integer(sizeof(uint64_t) * CHAR_BIT
,
549 lttng_alignof(uint64_t) * CHAR_BIT
,
550 lttng_is_signed_type(uint64_t),
554 goto type_alloc_error
;
556 field
= lttng_append_context(ctx
);
559 goto append_context_error
;
561 if (lttng_find_context(*ctx
, name_alloc
)) {
566 field
->destroy
= lttng_destroy_perf_counter_field
;
568 field
->event_field
->name
= name_alloc
;
569 field
->event_field
->type
= ust_type
;
570 field
->get_size
= perf_counter_get_size
;
571 field
->record
= perf_counter_record
;
572 field
->get_value
= perf_counter_get_value
;
574 perf_field
->attr
.type
= type
;
575 perf_field
->attr
.config
= config
;
576 perf_field
->attr
.exclude_kernel
= perf_get_exclude_kernel();
577 CDS_INIT_LIST_HEAD(&perf_field
->thread_field_list
);
578 field
->priv
= perf_field
;
580 /* Ensure that this perf counter can be used in this process. */
581 ret
= open_perf_fd(&perf_field
->attr
);
589 * Contexts can only be added before tracing is started, so we
590 * don't have to synchronize against concurrent threads using
594 lttng_context_update(*ctx
);
599 lttng_remove_context_field(ctx
, field
);
600 append_context_error
:
601 lttng_ust_destroy_type(ust_type
);
604 perf_field_alloc_error
:
610 int lttng_perf_counter_init(void)
614 ret
= pthread_key_create(&perf_counter_key
,
615 lttng_destroy_perf_thread_key
);
621 void lttng_perf_counter_exit(void)
625 ret
= pthread_key_delete(perf_counter_key
);
628 PERROR("Error in pthread_key_delete");