From: Mathieu Desnoyers Date: Fri, 7 Feb 2014 16:07:44 +0000 (-0500) Subject: Implement LTTng-UST perf counters support on x86 X-Git-Tag: v2.5.0-rc1~11 X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=d58d145447ffcc9497801769e1fe754b30dca5b8;p=lttng-ust.git Implement LTTng-UST perf counters support on x86 Signed-off-by: Mathieu Desnoyers --- diff --git a/configure.ac b/configure.ac index 894b2602..8bf7c040 100644 --- a/configure.ac +++ b/configure.ac @@ -27,6 +27,7 @@ AC_CONFIG_SRCDIR([include/lttng/tracepoint.h]) AC_CONFIG_HEADERS([config.h include/lttng/ust-config.h]) AH_TEMPLATE([LTTNG_UST_HAVE_EFFICIENT_UNALIGNED_ACCESS], [Use efficient unaligned access.]) AH_TEMPLATE([LTTNG_UST_HAVE_SDT_INTEGRATION], [SystemTap integration via sdt.h]) +AH_TEMPLATE([LTTNG_UST_HAVE_PERF_EVENT], [Perf event integration via perf_event.h]) # Compute minor/major/patchlevel version numbers AC_PROG_SED @@ -191,6 +192,14 @@ AC_CHECK_LIB([urcu-bp], [synchronize_rcu_bp], [], [AC_MSG_ERROR([Cannot find lib # urcu - check that URCU lib is at least version 0.6 AC_CHECK_LIB([urcu-bp], [call_rcu_bp], [], [AC_MSG_ERROR([liburcu 0.6 or newer is needed, please update your version or use [LDFLAGS]=-Ldir to specify the right location.])]) +# optional linux/perf_event.h +AC_CHECK_HEADERS([linux/perf_event.h], [have_perf_event=yes], []) +AM_CONDITIONAL([HAVE_PERF_EVENT], [test "x$have_perf_event" = "xyes"]) + +if test "x$have_perf_event" = "xyes"; then +AC_DEFINE([LTTNG_UST_HAVE_PERF_EVENT], [1]) +fi + AC_MSG_CHECKING([host system alignment requirements]) case $host_cpu in changequote(,)dnl diff --git a/include/lttng/ust-abi.h b/include/lttng/ust-abi.h index 0287466b..8c2469e4 100644 --- a/include/lttng/ust-abi.h +++ b/include/lttng/ust-abi.h @@ -138,8 +138,15 @@ enum lttng_ust_context_type { LTTNG_UST_CONTEXT_PTHREAD_ID = 2, LTTNG_UST_CONTEXT_PROCNAME = 3, LTTNG_UST_CONTEXT_IP = 4, + LTTNG_UST_CONTEXT_PERF_THREAD_COUNTER = 5, }; +struct lttng_ust_perf_counter_ctx { + uint32_t type; + uint64_t config; + char name[LTTNG_UST_SYM_NAME_LEN]; +} LTTNG_PACKED; + #define LTTNG_UST_CONTEXT_PADDING1 16 #define LTTNG_UST_CONTEXT_PADDING2 (LTTNG_UST_SYM_NAME_LEN + 32) struct lttng_ust_context { @@ -147,6 +154,7 @@ struct lttng_ust_context { char padding[LTTNG_UST_CONTEXT_PADDING1]; union { + struct lttng_ust_perf_counter_ctx perf_counter; char padding[LTTNG_UST_CONTEXT_PADDING2]; } u; } LTTNG_PACKED; diff --git a/include/lttng/ust-config.h.in b/include/lttng/ust-config.h.in index 5d1ee408..fb08f9f5 100644 --- a/include/lttng/ust-config.h.in +++ b/include/lttng/ust-config.h.in @@ -5,3 +5,6 @@ /* DTrace/GDB/SystemTap integration via sdt.h */ #undef LTTNG_UST_HAVE_SDT_INTEGRATION + +/* Perf event integration via perf_event.h */ +#undef LTTNG_UST_HAVE_PERF_EVENT diff --git a/include/lttng/ust-events.h b/include/lttng/ust-events.h index 3d473407..01f611b2 100644 --- a/include/lttng/ust-events.h +++ b/include/lttng/ust-events.h @@ -30,10 +30,14 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include #ifdef __cplusplus extern "C" { @@ -236,6 +240,8 @@ union lttng_ctx_value { double d; }; +struct lttng_perf_counter_field; + #define LTTNG_UST_CTX_FIELD_PADDING 40 struct lttng_ctx_field { struct lttng_event_field event_field; @@ -246,6 +252,7 @@ struct lttng_ctx_field { void (*get_value)(struct lttng_ctx_field *field, union lttng_ctx_value *value); union { + struct lttng_perf_counter_field *perf_counter; char padding[LTTNG_UST_CTX_FIELD_PADDING]; } u; void (*destroy)(struct lttng_ctx_field *field); @@ -594,6 +601,33 @@ int lttng_add_ip_to_ctx(struct lttng_ctx **ctx); void lttng_context_vtid_reset(void); void lttng_context_vpid_reset(void); +#ifdef LTTNG_UST_HAVE_PERF_EVENT +int lttng_add_perf_counter_to_ctx(uint32_t type, + uint64_t config, + const char *name, + struct lttng_ctx **ctx); +int lttng_perf_counter_init(void); +void lttng_perf_counter_exit(void); +#else /* #ifdef LTTNG_UST_HAVE_PERF_EVENT */ +static inline +int lttng_add_perf_counter_to_ctx(uint32_t type, + uint64_t config, + const char *name, + struct lttng_ctx **ctx) +{ + return -ENOSYS; +} +static inline +int lttng_perf_counter_init(void) +{ + return 0; +} +static inline +void lttng_perf_counter_exit(void) +{ +} +#endif /* #else #ifdef LTTNG_UST_HAVE_PERF_EVENT */ + extern const struct lttng_ust_client_lib_ring_buffer_client_cb *lttng_client_callbacks_metadata; extern const struct lttng_ust_client_lib_ring_buffer_client_cb *lttng_client_callbacks_discard; extern const struct lttng_ust_client_lib_ring_buffer_client_cb *lttng_client_callbacks_overwrite; diff --git a/liblttng-ust-ctl/ustctl.c b/liblttng-ust-ctl/ustctl.c index e07ec926..49ae3e6c 100644 --- a/liblttng-ust-ctl/ustctl.c +++ b/liblttng-ust-ctl/ustctl.c @@ -230,7 +230,7 @@ int ustctl_add_context(int sock, struct lttng_ust_context *ctx, memset(&lum, 0, sizeof(lum)); lum.handle = obj_data->handle; lum.cmd = LTTNG_UST_CONTEXT; - lum.u.context.ctx = ctx->ctx; + lum.u.context = *ctx; ret = ustcomm_send_app_cmd(sock, &lum, &lur); if (ret) { free(context_data); diff --git a/liblttng-ust/Makefile.am b/liblttng-ust/Makefile.am index ea4e403c..e2e1baaa 100644 --- a/liblttng-ust/Makefile.am +++ b/liblttng-ust/Makefile.am @@ -48,6 +48,11 @@ liblttng_ust_runtime_la_SOURCES = \ tracef.c \ lttng-ust-tracef-provider.h +if HAVE_PERF_EVENT +liblttng_ust_runtime_la_SOURCES += \ + lttng-context-perf-counters.c +endif + liblttng_ust_support_la_SOURCES = \ lttng-tracer.h \ lttng-tracer-core.h \ diff --git a/liblttng-ust/lttng-context-perf-counters.c b/liblttng-ust/lttng-context-perf-counters.c new file mode 100644 index 00000000..d6bd41cc --- /dev/null +++ b/liblttng-ust/lttng-context-perf-counters.c @@ -0,0 +1,435 @@ +/* + * lttng-context-perf-counters.c + * + * LTTng UST performance monitoring counters (perf-counters) integration. + * + * Copyright (C) 2009-2014 Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; only + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lttng-tracer-core.h" + +/* + * We use a global perf counter key and iterate on per-thread RCU lists + * of fields in the fast path, even though this is not strictly speaking + * what would provide the best fast-path complexity, to ensure teardown + * of sessions vs thread exit is handled racelessly. + * + * Updates and traversals of thread_list are protected by UST lock. + * Updates to rcu_field_list are protected by UST lock. + */ + +struct lttng_perf_counter_thread_field { + struct lttng_perf_counter_field *field; /* Back reference */ + struct perf_event_mmap_page *pc; + struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */ + struct cds_list_head rcu_field_node; /* RCU per-thread list of fields (node) */ +}; + +struct lttng_perf_counter_thread { + struct cds_list_head rcu_field_list; /* RCU per-thread list of fields */ +}; + +struct lttng_perf_counter_field { + struct perf_event_attr attr; + struct cds_list_head thread_field_list; /* Per-field list of thread fields */ +}; + +static pthread_key_t perf_counter_key; + +static +size_t perf_counter_get_size(size_t offset) +{ + size_t size = 0; + + size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t)); + size += sizeof(uint64_t); + return size; +} + +#if defined(__x86_64__) || defined(__i386__) + +static +uint64_t rdpmc(unsigned int counter) +{ + unsigned int low, high; + + asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter)); + + return low | ((uint64_t) high) << 32; +} + +#else /* defined(__x86_64__) || defined(__i386__) */ + +#error "Perf event counters are only supported on x86 so far." + +#endif /* #else defined(__x86_64__) || defined(__i386__) */ + +static +uint64_t read_perf_counter(struct perf_event_mmap_page *pc) +{ + uint32_t seq, idx; + uint64_t count; + + if (caa_unlikely(!pc)) + return 0; + + do { + seq = CMM_LOAD_SHARED(pc->lock); + cmm_barrier(); + + idx = pc->index; + if (idx) + count = pc->offset + rdpmc(idx - 1); + else + count = 0; + + cmm_barrier(); + } while (CMM_LOAD_SHARED(pc->lock) != seq); + + return count; +} + +static +int sys_perf_event_open(struct perf_event_attr *attr, + pid_t pid, int cpu, int group_fd, + unsigned long flags) +{ + return syscall(SYS_perf_event_open, attr, pid, cpu, + group_fd, flags); +} + +static +struct perf_event_mmap_page *setup_perf(struct perf_event_attr *attr) +{ + void *perf_addr; + int fd; + + fd = sys_perf_event_open(attr, 0, -1, -1, 0); + if (fd < 0) + return NULL; + + perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page), + PROT_READ, MAP_SHARED, fd, 0); + if (perf_addr == MAP_FAILED) + return NULL; + close(fd); + return perf_addr; +} + +static +void unmap_perf_page(struct perf_event_mmap_page *pc) +{ + int ret; + + if (!pc) + return; + ret = munmap(pc, sizeof(struct perf_event_mmap_page)); + if (ret < 0) { + PERROR("Error in munmap"); + abort(); + } +} + +static +struct lttng_perf_counter_thread *alloc_perf_counter_thread(void) +{ + struct lttng_perf_counter_thread *perf_thread; + sigset_t newmask, oldmask; + int ret; + + ret = sigfillset(&newmask); + if (ret) + abort(); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); + if (ret) + abort(); + /* Check again with signals disabled */ + perf_thread = pthread_getspecific(perf_counter_key); + if (perf_thread) + goto skip; + perf_thread = zmalloc(sizeof(*perf_thread)); + if (!perf_thread) + abort(); + CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list); + ret = pthread_setspecific(perf_counter_key, perf_thread); + if (ret) + abort(); +skip: + ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); + if (ret) + abort(); + return perf_thread; +} + +static +struct lttng_perf_counter_thread_field * + add_thread_field(struct lttng_perf_counter_field *perf_field, + struct lttng_perf_counter_thread *perf_thread) +{ + struct lttng_perf_counter_thread_field *thread_field; + sigset_t newmask, oldmask; + int ret; + + ret = sigfillset(&newmask); + if (ret) + abort(); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); + if (ret) + abort(); + /* Check again with signals disabled */ + cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list, + rcu_field_node) { + if (thread_field->field == perf_field) + goto skip; + } + thread_field = zmalloc(sizeof(*thread_field)); + if (!thread_field) + abort(); + thread_field->field = perf_field; + thread_field->pc = setup_perf(&perf_field->attr); + /* Note: thread_field->pc can be NULL if setup_perf() fails. */ + ust_lock_nocheck(); + cds_list_add_rcu(&thread_field->rcu_field_node, + &perf_thread->rcu_field_list); + cds_list_add(&thread_field->thread_field_node, + &perf_field->thread_field_list); + ust_unlock(); +skip: + ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); + if (ret) + abort(); + return thread_field; +} + +static +struct lttng_perf_counter_thread_field * + get_thread_field(struct lttng_perf_counter_field *field) +{ + struct lttng_perf_counter_thread *perf_thread; + struct lttng_perf_counter_thread_field *thread_field; + + perf_thread = pthread_getspecific(perf_counter_key); + if (!perf_thread) + perf_thread = alloc_perf_counter_thread(); + cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list, + rcu_field_node) { + if (thread_field->field == field) + return thread_field; + } + /* perf_counter_thread_field not found, need to add one */ + return add_thread_field(field, perf_thread); +} + +static +uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field) +{ + struct lttng_perf_counter_field *perf_field; + struct lttng_perf_counter_thread_field *perf_thread_field; + + perf_field = field->u.perf_counter; + perf_thread_field = get_thread_field(perf_field); + return read_perf_counter(perf_thread_field->pc); +} + +static +void perf_counter_record(struct lttng_ctx_field *field, + struct lttng_ust_lib_ring_buffer_ctx *ctx, + struct lttng_channel *chan) +{ + uint64_t value; + + value = wrapper_perf_counter_read(field); + lib_ring_buffer_align_ctx(ctx, lttng_alignof(value)); + chan->ops->event_write(ctx, &value, sizeof(value)); +} + +static +void perf_counter_get_value(struct lttng_ctx_field *field, + union lttng_ctx_value *value) +{ + uint64_t v; + + v = wrapper_perf_counter_read(field); + value->s64 = v; +} + +/* Called with UST lock held */ +static +void lttng_destroy_perf_thread_field( + struct lttng_perf_counter_thread_field *thread_field) +{ + unmap_perf_page(thread_field->pc); + cds_list_del_rcu(&thread_field->rcu_field_node); + cds_list_del(&thread_field->thread_field_node); + free(thread_field); +} + +static +void lttng_destroy_perf_thread_key(void *_key) +{ + struct lttng_perf_counter_thread *perf_thread = _key; + struct lttng_perf_counter_thread_field *pos, *p; + + ust_lock_nocheck(); + cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list, + rcu_field_node) + lttng_destroy_perf_thread_field(pos); + ust_unlock(); + free(perf_thread); +} + +/* Called with UST lock held */ +static +void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field) +{ + struct lttng_perf_counter_field *perf_field; + struct lttng_perf_counter_thread_field *pos, *p; + + free((char *) field->event_field.name); + perf_field = field->u.perf_counter; + /* + * This put is performed when no threads can concurrently + * perform a "get" concurrently, thanks to urcu-bp grace + * period. + */ + cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list, + thread_field_node) + lttng_destroy_perf_thread_field(pos); + free(perf_field); +} + +/* Called with UST lock held */ +int lttng_add_perf_counter_to_ctx(uint32_t type, + uint64_t config, + const char *name, + struct lttng_ctx **ctx) +{ + struct lttng_ctx_field *field; + struct lttng_perf_counter_field *perf_field; + struct perf_event_mmap_page *tmp_pc; + char *name_alloc; + int ret; + + name_alloc = strdup(name); + if (!name_alloc) { + ret = -ENOMEM; + goto name_alloc_error; + } + perf_field = zmalloc(sizeof(*perf_field)); + if (!perf_field) { + ret = -ENOMEM; + goto perf_field_alloc_error; + } + field = lttng_append_context(ctx); + if (!field) { + ret = -ENOMEM; + goto append_context_error; + } + if (lttng_find_context(*ctx, name_alloc)) { + ret = -EEXIST; + goto find_error; + } + + field->destroy = lttng_destroy_perf_counter_field; + + field->event_field.name = name_alloc; + field->event_field.type.atype = atype_integer; + field->event_field.type.u.basic.integer.size = + sizeof(uint64_t) * CHAR_BIT; + field->event_field.type.u.basic.integer.alignment = + lttng_alignof(uint64_t) * CHAR_BIT; + field->event_field.type.u.basic.integer.signedness = + lttng_is_signed_type(uint64_t); + field->event_field.type.u.basic.integer.reverse_byte_order = 0; + field->event_field.type.u.basic.integer.base = 10; + field->event_field.type.u.basic.integer.encoding = lttng_encode_none; + field->get_size = perf_counter_get_size; + field->record = perf_counter_record; + field->get_value = perf_counter_get_value; + + perf_field->attr.type = type; + perf_field->attr.config = config; + perf_field->attr.exclude_kernel = 1; + CDS_INIT_LIST_HEAD(&perf_field->thread_field_list); + field->u.perf_counter = perf_field; + + /* Ensure that this perf counter can be used in this process. */ + tmp_pc = setup_perf(&perf_field->attr); + if (!tmp_pc) { + ret = -ENODEV; + goto setup_error; + } + unmap_perf_page(tmp_pc); + + /* + * Contexts can only be added before tracing is started, so we + * don't have to synchronize against concurrent threads using + * the field here. + */ + + return 0; + +setup_error: +find_error: + lttng_remove_context_field(ctx, field); +append_context_error: + free(perf_field); +perf_field_alloc_error: + free(name_alloc); +name_alloc_error: + return ret; +} + +int lttng_perf_counter_init(void) +{ + int ret; + + ret = pthread_key_create(&perf_counter_key, + lttng_destroy_perf_thread_key); + if (ret) + ret = -ret; + return ret; +} + +void lttng_perf_counter_exit(void) +{ + int ret; + + ret = pthread_key_delete(perf_counter_key); + if (ret) { + errno = ret; + PERROR("Error in pthread_key_delete"); + } +} diff --git a/liblttng-ust/lttng-events.c b/liblttng-ust/lttng-events.c index 6dc7bfca..0cab2f49 100644 --- a/liblttng-ust/lttng-events.c +++ b/liblttng-ust/lttng-events.c @@ -797,6 +797,17 @@ int lttng_attach_context(struct lttng_ust_context *context_param, switch (context_param->ctx) { case LTTNG_UST_CONTEXT_PTHREAD_ID: return lttng_add_pthread_id_to_ctx(ctx); + case LTTNG_UST_CONTEXT_PERF_THREAD_COUNTER: + { + struct lttng_ust_perf_counter_ctx *perf_ctx_param; + + perf_ctx_param = &context_param->u.perf_counter; + return lttng_add_perf_counter_to_ctx( + perf_ctx_param->type, + perf_ctx_param->config, + perf_ctx_param->name, + ctx); + } case LTTNG_UST_CONTEXT_VTID: return lttng_add_vtid_to_ctx(ctx); case LTTNG_UST_CONTEXT_VPID: diff --git a/liblttng-ust/lttng-ust-comm.c b/liblttng-ust/lttng-ust-comm.c index 77cde295..5df33f5f 100644 --- a/liblttng-ust/lttng-ust-comm.c +++ b/liblttng-ust/lttng-ust-comm.c @@ -67,9 +67,16 @@ static int initialized; * probe registration. * * ust_exit_mutex must never nest in ust_mutex. + * + * ust_mutex_nest is a per-thread nesting counter, allowing the perf + * counter lazy initialization called by events within the statedump, + * which traces while the ust_mutex is held. */ static pthread_mutex_t ust_mutex = PTHREAD_MUTEX_INITIALIZER; +/* Allow nesting the ust_mutex within the same thread. */ +static DEFINE_URCU_TLS(int, ust_mutex_nest); + /* * ust_exit_mutex protects thread_active variable wrt thread exit. It * cannot be done by ust_mutex because pthread_cancel(), which takes an @@ -83,12 +90,26 @@ static pthread_mutex_t ust_exit_mutex = PTHREAD_MUTEX_INITIALIZER; static int lttng_ust_comm_should_quit; /* - * Return 0 on success, -1 if should quilt. + * Return 0 on success, -1 if should quit. * The lock is taken in both cases. + * Signal-safe. */ int ust_lock(void) { - pthread_mutex_lock(&ust_mutex); + sigset_t sig_all_blocked, orig_mask; + int ret; + + sigfillset(&sig_all_blocked); + ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } + if (!URCU_TLS(ust_mutex_nest)++) + pthread_mutex_lock(&ust_mutex); + ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } if (lttng_ust_comm_should_quit) { return -1; } else { @@ -100,15 +121,45 @@ int ust_lock(void) * ust_lock_nocheck() can be used in constructors/destructors, because * they are already nested within the dynamic loader lock, and therefore * have exclusive access against execution of liblttng-ust destructor. + * Signal-safe. */ void ust_lock_nocheck(void) { - pthread_mutex_lock(&ust_mutex); + sigset_t sig_all_blocked, orig_mask; + int ret; + + sigfillset(&sig_all_blocked); + ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } + if (!URCU_TLS(ust_mutex_nest)++) + pthread_mutex_lock(&ust_mutex); + ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } } +/* + * Signal-safe. + */ void ust_unlock(void) { - pthread_mutex_unlock(&ust_mutex); + sigset_t sig_all_blocked, orig_mask; + int ret; + + sigfillset(&sig_all_blocked); + ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } + if (!--URCU_TLS(ust_mutex_nest)) + pthread_mutex_unlock(&ust_mutex); + ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL); + if (ret) { + ERR("pthread_sigmask: %s", strerror(ret)); + } } /* @@ -265,6 +316,12 @@ void lttng_fixup_nest_count_tls(void) asm volatile ("" : : "m" (URCU_TLS(lttng_ust_nest_count))); } +static +void lttng_fixup_ust_mutex_nest_tls(void) +{ + asm volatile ("" : : "m" (URCU_TLS(ust_mutex_nest))); +} + int lttng_get_notify_socket(void *owner) { struct sock_info *info = owner; @@ -1351,6 +1408,7 @@ void __attribute__((constructor)) lttng_ust_init(void) lttng_fixup_vtid_tls(); lttng_fixup_nest_count_tls(); lttng_fixup_procname_tls(); + lttng_fixup_ust_mutex_nest_tls(); /* * We want precise control over the order in which we construct @@ -1366,6 +1424,7 @@ void __attribute__((constructor)) lttng_ust_init(void) lttng_ring_buffer_client_overwrite_rt_init(); lttng_ring_buffer_client_discard_init(); lttng_ring_buffer_client_discard_rt_init(); + lttng_perf_counter_init(); lttng_context_init(); /* * Invoke ust malloc wrapper init before starting other threads. @@ -1474,6 +1533,7 @@ void lttng_ust_cleanup(int exiting) lttng_ust_abi_exit(); lttng_ust_events_exit(); lttng_context_exit(); + lttng_perf_counter_exit(); lttng_ring_buffer_client_discard_rt_exit(); lttng_ring_buffer_client_discard_exit(); lttng_ring_buffer_client_overwrite_rt_exit();