2 * SPDX-License-Identifier: LGPL-2.1-only
4 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * LTTng UST performance monitoring counters (perf-counters) integration.
11 #include <sys/types.h>
20 #include <sys/syscall.h>
21 #include <lttng/ust-arch.h>
22 #include <lttng/ust-events.h>
23 #include <lttng/ust-tracer.h>
24 #include <lttng/ust-ringbuffer-context.h>
25 #include <lttng/ust-cancelstate.h>
26 #include <urcu/system.h>
27 #include <urcu/arch.h>
28 #include <urcu/rculist.h>
29 #include "common/macros.h"
31 #include "common/logging.h"
33 #include <urcu/tls-compat.h>
34 #include "perf_event.h"
36 #include "context-internal.h"
37 #include "lttng-tracer-core.h"
38 #include "lib/lttng-ust/events.h"
41 * We use a global perf counter key and iterate on per-thread RCU lists
42 * of fields in the fast path, even though this is not strictly speaking
43 * what would provide the best fast-path complexity, to ensure teardown
44 * of sessions vs thread exit is handled racelessly.
46 * Updates and traversals of thread_list are protected by UST lock.
47 * Updates to rcu_field_list are protected by UST lock.
50 struct lttng_perf_counter_thread_field
{
51 struct lttng_perf_counter_field
*field
; /* Back reference */
52 struct perf_event_mmap_page
*pc
;
53 struct cds_list_head thread_field_node
; /* Per-field list of thread fields (node) */
54 struct cds_list_head rcu_field_node
; /* RCU per-thread list of fields (node) */
58 struct lttng_perf_counter_thread
{
59 struct cds_list_head rcu_field_list
; /* RCU per-thread list of fields */
62 struct lttng_perf_counter_field
{
63 struct perf_event_attr attr
;
64 struct cds_list_head thread_field_list
; /* Per-field list of thread fields */
66 struct lttng_ust_event_field
*event_field
;
69 static pthread_key_t perf_counter_key
;
72 * lttng_perf_lock - Protect lttng-ust perf counter data structures
74 * Nests within the ust_lock, and therefore within the libc dl lock.
75 * Therefore, we need to allocate the TLS before nesting into this lock.
76 * Nests inside RCU bp read-side lock. Protects against concurrent
79 static pthread_mutex_t ust_perf_mutex
= PTHREAD_MUTEX_INITIALIZER
;
82 * Track whether we are tracing from a signal handler nested on an
85 static DEFINE_URCU_TLS(int, ust_perf_mutex_nest
);
88 * Force a read (imply TLS allocation for dlopen) of TLS variables.
90 void lttng_ust_perf_counter_init_thread(int flags
)
92 __asm__
__volatile__ ("" : : "m" (URCU_TLS(ust_perf_mutex_nest
)));
96 void lttng_perf_lock(void)
98 sigset_t sig_all_blocked
, orig_mask
;
101 if (lttng_ust_cancelstate_disable_push()) {
102 ERR("lttng_ust_cancelstate_disable_push");
104 sigfillset(&sig_all_blocked
);
105 ret
= pthread_sigmask(SIG_SETMASK
, &sig_all_blocked
, &orig_mask
);
107 ERR("pthread_sigmask: %s", strerror(ret
));
109 if (!URCU_TLS(ust_perf_mutex_nest
)++) {
111 * Ensure the compiler don't move the store after the close()
112 * call in case close() would be marked as leaf.
115 pthread_mutex_lock(&ust_perf_mutex
);
117 ret
= pthread_sigmask(SIG_SETMASK
, &orig_mask
, NULL
);
119 ERR("pthread_sigmask: %s", strerror(ret
));
123 void lttng_perf_unlock(void)
125 sigset_t sig_all_blocked
, orig_mask
;
128 sigfillset(&sig_all_blocked
);
129 ret
= pthread_sigmask(SIG_SETMASK
, &sig_all_blocked
, &orig_mask
);
131 ERR("pthread_sigmask: %s", strerror(ret
));
134 * Ensure the compiler don't move the store before the close()
135 * call, in case close() would be marked as leaf.
138 if (!--URCU_TLS(ust_perf_mutex_nest
)) {
139 pthread_mutex_unlock(&ust_perf_mutex
);
141 ret
= pthread_sigmask(SIG_SETMASK
, &orig_mask
, NULL
);
143 ERR("pthread_sigmask: %s", strerror(ret
));
145 if (lttng_ust_cancelstate_disable_pop()) {
146 ERR("lttng_ust_cancelstate_disable_pop");
151 size_t perf_counter_get_size(void *priv
__attribute__((unused
)),
152 struct lttng_ust_probe_ctx
*probe_ctx
__attribute__((unused
)),
157 size
+= lttng_ust_ring_buffer_align(offset
, lttng_ust_rb_alignof(uint64_t));
158 size
+= sizeof(uint64_t);
163 uint64_t read_perf_counter_syscall(
164 struct lttng_perf_counter_thread_field
*thread_field
)
168 if (caa_unlikely(thread_field
->fd
< 0))
171 if (caa_unlikely(read(thread_field
->fd
, &count
, sizeof(count
))
178 #if defined(LTTNG_UST_ARCH_X86)
181 uint64_t rdpmc(unsigned int counter
)
183 unsigned int low
, high
;
185 __asm__
__volatile__("rdpmc" : "=a" (low
), "=d" (high
) : "c" (counter
));
187 return low
| ((uint64_t) high
) << 32;
191 bool has_rdpmc(struct perf_event_mmap_page
*pc
)
193 if (caa_unlikely(!pc
->cap_bit0_is_deprecated
))
195 /* Since Linux kernel 3.12. */
196 return pc
->cap_user_rdpmc
;
200 uint64_t arch_read_perf_counter(
201 struct lttng_perf_counter_thread_field
*thread_field
)
205 struct perf_event_mmap_page
*pc
= thread_field
->pc
;
207 if (caa_unlikely(!pc
))
211 seq
= CMM_LOAD_SHARED(pc
->lock
);
215 if (caa_likely(has_rdpmc(pc
) && idx
)) {
218 pmcval
= rdpmc(idx
- 1);
219 /* Sign-extend the pmc register result. */
220 pmcval
<<= 64 - pc
->pmc_width
;
221 pmcval
>>= 64 - pc
->pmc_width
;
222 count
= pc
->offset
+ pmcval
;
224 /* Fall-back on system call if rdpmc cannot be used. */
225 return read_perf_counter_syscall(thread_field
);
228 } while (CMM_LOAD_SHARED(pc
->lock
) != seq
);
234 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field
*thread_field
)
236 struct perf_event_mmap_page
*pc
= thread_field
->pc
;
240 return !has_rdpmc(pc
);
245 /* Generic (slow) implementation using a read system call. */
247 uint64_t arch_read_perf_counter(
248 struct lttng_perf_counter_thread_field
*thread_field
)
250 return read_perf_counter_syscall(thread_field
);
254 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field
*thread_field
__attribute__((unused
)))
262 int sys_perf_event_open(struct perf_event_attr
*attr
,
263 pid_t pid
, int cpu
, int group_fd
,
266 return syscall(SYS_perf_event_open
, attr
, pid
, cpu
,
271 int open_perf_fd(struct perf_event_attr
*attr
)
275 fd
= sys_perf_event_open(attr
, 0, -1, -1, 0);
283 void close_perf_fd(int fd
)
292 perror("Error closing LTTng-UST perf memory mapping FD");
296 static void setup_perf(struct lttng_perf_counter_thread_field
*thread_field
)
300 perf_addr
= mmap(NULL
, sizeof(struct perf_event_mmap_page
),
301 PROT_READ
, MAP_SHARED
, thread_field
->fd
, 0);
302 if (perf_addr
== MAP_FAILED
)
304 thread_field
->pc
= perf_addr
;
306 if (!arch_perf_keep_fd(thread_field
)) {
307 close_perf_fd(thread_field
->fd
);
308 thread_field
->fd
= -1;
313 void unmap_perf_page(struct perf_event_mmap_page
*pc
)
319 ret
= munmap(pc
, sizeof(struct perf_event_mmap_page
));
321 PERROR("Error in munmap");
327 struct lttng_perf_counter_thread
*alloc_perf_counter_thread(void)
329 struct lttng_perf_counter_thread
*perf_thread
;
330 sigset_t newmask
, oldmask
;
333 ret
= sigfillset(&newmask
);
336 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
339 /* Check again with signals disabled */
340 perf_thread
= pthread_getspecific(perf_counter_key
);
343 perf_thread
= zmalloc(sizeof(*perf_thread
));
346 CDS_INIT_LIST_HEAD(&perf_thread
->rcu_field_list
);
347 ret
= pthread_setspecific(perf_counter_key
, perf_thread
);
351 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
358 struct lttng_perf_counter_thread_field
*
359 add_thread_field(struct lttng_perf_counter_field
*perf_field
,
360 struct lttng_perf_counter_thread
*perf_thread
)
362 struct lttng_perf_counter_thread_field
*thread_field
;
363 sigset_t newmask
, oldmask
;
366 ret
= sigfillset(&newmask
);
369 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
372 /* Check again with signals disabled */
373 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
375 if (thread_field
->field
== perf_field
)
378 thread_field
= zmalloc(sizeof(*thread_field
));
381 thread_field
->field
= perf_field
;
382 thread_field
->fd
= open_perf_fd(&perf_field
->attr
);
383 if (thread_field
->fd
>= 0)
384 setup_perf(thread_field
);
386 * Note: thread_field->pc can be NULL if setup_perf() fails.
387 * Also, thread_field->fd can be -1 if open_perf_fd() fails.
390 cds_list_add_rcu(&thread_field
->rcu_field_node
,
391 &perf_thread
->rcu_field_list
);
392 cds_list_add(&thread_field
->thread_field_node
,
393 &perf_field
->thread_field_list
);
396 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
403 struct lttng_perf_counter_thread_field
*
404 get_thread_field(struct lttng_perf_counter_field
*field
)
406 struct lttng_perf_counter_thread
*perf_thread
;
407 struct lttng_perf_counter_thread_field
*thread_field
;
409 perf_thread
= pthread_getspecific(perf_counter_key
);
411 perf_thread
= alloc_perf_counter_thread();
412 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
414 if (thread_field
->field
== field
)
417 /* perf_counter_thread_field not found, need to add one */
418 return add_thread_field(field
, perf_thread
);
422 uint64_t wrapper_perf_counter_read(void *priv
)
424 struct lttng_perf_counter_field
*perf_field
;
425 struct lttng_perf_counter_thread_field
*perf_thread_field
;
427 perf_field
= (struct lttng_perf_counter_field
*) priv
;
428 perf_thread_field
= get_thread_field(perf_field
);
429 return arch_read_perf_counter(perf_thread_field
);
433 void perf_counter_record(void *priv
,
434 struct lttng_ust_probe_ctx
*probe_ctx
__attribute__((unused
)),
435 struct lttng_ust_ring_buffer_ctx
*ctx
,
436 struct lttng_ust_channel_buffer
*chan
)
440 value
= wrapper_perf_counter_read(priv
);
441 chan
->ops
->event_write(ctx
, &value
, sizeof(value
), lttng_ust_rb_alignof(value
));
445 void perf_counter_get_value(void *priv
,
446 struct lttng_ust_probe_ctx
*probe_ctx
__attribute__((unused
)),
447 struct lttng_ust_ctx_value
*value
)
449 value
->u
.u64
= wrapper_perf_counter_read(priv
);
452 /* Called with perf lock held */
454 void lttng_destroy_perf_thread_field(
455 struct lttng_perf_counter_thread_field
*thread_field
)
457 close_perf_fd(thread_field
->fd
);
458 unmap_perf_page(thread_field
->pc
);
459 cds_list_del_rcu(&thread_field
->rcu_field_node
);
460 cds_list_del(&thread_field
->thread_field_node
);
465 void lttng_destroy_perf_thread_key(void *_key
)
467 struct lttng_perf_counter_thread
*perf_thread
= _key
;
468 struct lttng_perf_counter_thread_field
*pos
, *p
;
471 cds_list_for_each_entry_safe(pos
, p
, &perf_thread
->rcu_field_list
,
473 lttng_destroy_perf_thread_field(pos
);
478 /* Called with UST lock held */
480 void lttng_destroy_perf_counter_ctx_field(void *priv
)
482 struct lttng_perf_counter_field
*perf_field
;
483 struct lttng_perf_counter_thread_field
*pos
, *p
;
485 perf_field
= (struct lttng_perf_counter_field
*) priv
;
486 free(perf_field
->name
);
488 * This put is performed when no threads can concurrently
489 * perform a "get" concurrently, thanks to urcu-bp grace
490 * period. Holding the lttng perf lock protects against
491 * concurrent modification of the per-thread thread field
495 cds_list_for_each_entry_safe(pos
, p
, &perf_field
->thread_field_list
,
497 lttng_destroy_perf_thread_field(pos
);
499 free(perf_field
->event_field
);
503 #ifdef LTTNG_UST_ARCH_ARMV7
506 int perf_get_exclude_kernel(void)
511 #else /* LTTNG_UST_ARCH_ARMV7 */
514 int perf_get_exclude_kernel(void)
519 #endif /* LTTNG_UST_ARCH_ARMV7 */
521 static const struct lttng_ust_type_common
*ust_type
=
522 lttng_ust_static_type_integer(sizeof(uint64_t) * CHAR_BIT
,
523 lttng_ust_rb_alignof(uint64_t) * CHAR_BIT
,
524 lttng_ust_is_signed_type(uint64_t),
525 LTTNG_UST_BYTE_ORDER
, 10);
527 /* Called with UST lock held */
528 int lttng_add_perf_counter_to_ctx(uint32_t type
,
531 struct lttng_ust_ctx
**ctx
)
533 struct lttng_ust_ctx_field ctx_field
;
534 struct lttng_ust_event_field
*event_field
;
535 struct lttng_perf_counter_field
*perf_field
;
539 if (lttng_find_context(*ctx
, name
)) {
543 name_alloc
= strdup(name
);
546 goto name_alloc_error
;
548 event_field
= zmalloc(sizeof(*event_field
));
551 goto event_field_alloc_error
;
553 event_field
->name
= name_alloc
;
554 event_field
->type
= ust_type
;
556 perf_field
= zmalloc(sizeof(*perf_field
));
559 goto perf_field_alloc_error
;
561 perf_field
->attr
.type
= type
;
562 perf_field
->attr
.config
= config
;
563 perf_field
->attr
.exclude_kernel
= perf_get_exclude_kernel();
564 CDS_INIT_LIST_HEAD(&perf_field
->thread_field_list
);
565 perf_field
->name
= name_alloc
;
566 perf_field
->event_field
= event_field
;
568 /* Ensure that this perf counter can be used in this process. */
569 ret
= open_perf_fd(&perf_field
->attr
);
576 ctx_field
.event_field
= event_field
;
577 ctx_field
.get_size
= perf_counter_get_size
;
578 ctx_field
.record
= perf_counter_record
;
579 ctx_field
.get_value
= perf_counter_get_value
;
580 ctx_field
.destroy
= lttng_destroy_perf_counter_ctx_field
;
581 ctx_field
.priv
= perf_field
;
583 ret
= lttng_ust_context_append(ctx
, &ctx_field
);
586 goto append_context_error
;
590 append_context_error
:
593 perf_field_alloc_error
:
595 event_field_alloc_error
:
602 int lttng_perf_counter_init(void)
606 ret
= pthread_key_create(&perf_counter_key
,
607 lttng_destroy_perf_thread_key
);
613 void lttng_perf_counter_exit(void)
617 ret
= pthread_key_delete(perf_counter_key
);
620 PERROR("Error in pthread_key_delete");