lttng-context-perf-counters.c

   1 /* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
   2  *
   3  * lttng-context-perf-counters.c
   4  *
   5  * LTTng performance monitoring counters (perf-counters) integration module.
   6  *
   7  * Copyright (C) 2009-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/slab.h>
  12 #include <linux/perf_event.h>
  13 #include <linux/list.h>
  14 #include <linux/string.h>
  15 #include <linux/cpu.h>
  16 #include <lttng-events.h>
  17 #include <wrapper/ringbuffer/frontend_types.h>
  18 #include <wrapper/vmalloc.h>
  19 #include <wrapper/perf.h>
  20 #include <lttng-tracer.h>
  21
  22 static
  23 size_t perf_counter_get_size(size_t offset)
  24 {
  25         size_t size = 0;
  26
  27         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  28         size += sizeof(uint64_t);
  29         return size;
  30 }
  31
  32 static
  33 void perf_counter_record(struct lttng_ctx_field *field,
  34                          struct lib_ring_buffer_ctx *ctx,
  35                          struct lttng_channel *chan)
  36 {
  37         struct perf_event *event;
  38         uint64_t value;
  39
  40         event = field->u.perf_counter->e[ctx->cpu];
  41         if (likely(event)) {
  42                 if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
  43                         value = 0;
  44                 } else {
  45                         event->pmu->read(event);
  46                         value = local64_read(&event->count);
  47                 }
  48         } else {
  49                 /*
  50                  * Perf chooses not to be clever and not to support enabling a
  51                  * perf counter before the cpu is brought up. Therefore, we need
  52                  * to support having events coming (e.g. scheduler events)
  53                  * before the counter is setup. Write an arbitrary 0 in this
  54                  * case.
  55                  */
  56                 value = 0;
  57         }
  58         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
  59         chan->ops->event_write(ctx, &value, sizeof(value));
  60 }
  61
  62 #if defined(CONFIG_PERF_EVENTS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,99))
  63 static
  64 void overflow_callback(struct perf_event *event,
  65                        struct perf_sample_data *data,
  66                        struct pt_regs *regs)
  67 {
  68 }
  69 #else
  70 static
  71 void overflow_callback(struct perf_event *event, int nmi,
  72                        struct perf_sample_data *data,
  73                        struct pt_regs *regs)
  74 {
  75 }
  76 #endif
  77
  78 static
  79 void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
  80 {
  81         struct perf_event **events = field->u.perf_counter->e;
  82
  83 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
  84         {
  85                 int ret;
  86
  87                 ret = cpuhp_state_remove_instance(lttng_hp_online,
  88                         &field->u.perf_counter->cpuhp_online.node);
  89                 WARN_ON(ret);
  90                 ret = cpuhp_state_remove_instance(lttng_hp_prepare,
  91                         &field->u.perf_counter->cpuhp_prepare.node);
  92                 WARN_ON(ret);
  93         }
  94 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
  95         {
  96                 int cpu;
  97
  98                 get_online_cpus();
  99                 for_each_online_cpu(cpu)
 100                         perf_event_release_kernel(events[cpu]);
 101                 put_online_cpus();
 102 #ifdef CONFIG_HOTPLUG_CPU
 103                 unregister_cpu_notifier(&field->u.perf_counter->nb);
 104 #endif
 105         }
 106 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 107         kfree(field->event_field.name);
 108         kfree(field->u.perf_counter->attr);
 109         lttng_kvfree(events);
 110         kfree(field->u.perf_counter);
 111 }
 112
 113 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
 114
 115 int lttng_cpuhp_perf_counter_online(unsigned int cpu,
 116                 struct lttng_cpuhp_node *node)
 117 {
 118         struct lttng_perf_counter_field *perf_field =
 119                 container_of(node, struct lttng_perf_counter_field,
 120                                 cpuhp_online);
 121         struct perf_event **events = perf_field->e;
 122         struct perf_event_attr *attr = perf_field->attr;
 123         struct perf_event *pevent;
 124
 125         pevent = wrapper_perf_event_create_kernel_counter(attr,
 126                         cpu, NULL, overflow_callback);
 127         if (!pevent || IS_ERR(pevent))
 128                 return -EINVAL;
 129         if (pevent->state == PERF_EVENT_STATE_ERROR) {
 130                 perf_event_release_kernel(pevent);
 131                 return -EINVAL;
 132         }
 133         barrier();      /* Create perf counter before setting event */
 134         events[cpu] = pevent;
 135         return 0;
 136 }
 137
 138 int lttng_cpuhp_perf_counter_dead(unsigned int cpu,
 139                 struct lttng_cpuhp_node *node)
 140 {
 141         struct lttng_perf_counter_field *perf_field =
 142                 container_of(node, struct lttng_perf_counter_field,
 143                                 cpuhp_prepare);
 144         struct perf_event **events = perf_field->e;
 145         struct perf_event *pevent;
 146
 147         pevent = events[cpu];
 148         events[cpu] = NULL;
 149         barrier();      /* NULLify event before perf counter teardown */
 150         perf_event_release_kernel(pevent);
 151         return 0;
 152 }
 153
 154 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 155
 156 #ifdef CONFIG_HOTPLUG_CPU
 157
 158 /**
 159  *      lttng_perf_counter_hp_callback - CPU hotplug callback
 160  *      @nb: notifier block
 161  *      @action: hotplug action to take
 162  *      @hcpu: CPU number
 163  *
 164  *      Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
 165  *
 166  * We can setup perf counters when the cpu is online (up prepare seems to be too
 167  * soon).
 168  */
 169 static
 170 int lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
 171                                                  unsigned long action,
 172                                                  void *hcpu)
 173 {
 174         unsigned int cpu = (unsigned long) hcpu;
 175         struct lttng_perf_counter_field *perf_field =
 176                 container_of(nb, struct lttng_perf_counter_field, nb);
 177         struct perf_event **events = perf_field->e;
 178         struct perf_event_attr *attr = perf_field->attr;
 179         struct perf_event *pevent;
 180
 181         if (!perf_field->hp_enable)
 182                 return NOTIFY_OK;
 183
 184         switch (action) {
 185         case CPU_ONLINE:
 186         case CPU_ONLINE_FROZEN:
 187                 pevent = wrapper_perf_event_create_kernel_counter(attr,
 188                                 cpu, NULL, overflow_callback);
 189                 if (!pevent || IS_ERR(pevent))
 190                         return NOTIFY_BAD;
 191                 if (pevent->state == PERF_EVENT_STATE_ERROR) {
 192                         perf_event_release_kernel(pevent);
 193                         return NOTIFY_BAD;
 194                 }
 195                 barrier();      /* Create perf counter before setting event */
 196                 events[cpu] = pevent;
 197                 break;
 198         case CPU_UP_CANCELED:
 199         case CPU_UP_CANCELED_FROZEN:
 200         case CPU_DEAD:
 201         case CPU_DEAD_FROZEN:
 202                 pevent = events[cpu];
 203                 events[cpu] = NULL;
 204                 barrier();      /* NULLify event before perf counter teardown */
 205                 perf_event_release_kernel(pevent);
 206                 break;
 207         }
 208         return NOTIFY_OK;
 209 }
 210
 211 #endif
 212
 213 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 214
 215 int lttng_add_perf_counter_to_ctx(uint32_t type,
 216                                   uint64_t config,
 217                                   const char *name,
 218                                   struct lttng_ctx **ctx)
 219 {
 220         struct lttng_ctx_field *field;
 221         struct lttng_perf_counter_field *perf_field;
 222         struct perf_event **events;
 223         struct perf_event_attr *attr;
 224         int ret;
 225         char *name_alloc;
 226
 227         events = lttng_kvzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
 228         if (!events)
 229                 return -ENOMEM;
 230
 231         attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
 232         if (!attr) {
 233                 ret = -ENOMEM;
 234                 goto error_attr;
 235         }
 236
 237         attr->type = type;
 238         attr->config = config;
 239         attr->size = sizeof(struct perf_event_attr);
 240         attr->pinned = 1;
 241         attr->disabled = 0;
 242
 243         perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
 244         if (!perf_field) {
 245                 ret = -ENOMEM;
 246                 goto error_alloc_perf_field;
 247         }
 248         perf_field->e = events;
 249         perf_field->attr = attr;
 250
 251         name_alloc = kstrdup(name, GFP_KERNEL);
 252         if (!name_alloc) {
 253                 ret = -ENOMEM;
 254                 goto name_alloc_error;
 255         }
 256
 257         field = lttng_append_context(ctx);
 258         if (!field) {
 259                 ret = -ENOMEM;
 260                 goto append_context_error;
 261         }
 262         if (lttng_find_context(*ctx, name_alloc)) {
 263                 ret = -EEXIST;
 264                 goto find_error;
 265         }
 266
 267 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
 268
 269         perf_field->cpuhp_prepare.component = LTTNG_CONTEXT_PERF_COUNTERS;
 270         ret = cpuhp_state_add_instance(lttng_hp_prepare,
 271                 &perf_field->cpuhp_prepare.node);
 272         if (ret)
 273                 goto cpuhp_prepare_error;
 274
 275         perf_field->cpuhp_online.component = LTTNG_CONTEXT_PERF_COUNTERS;
 276         ret = cpuhp_state_add_instance(lttng_hp_online,
 277                 &perf_field->cpuhp_online.node);
 278         if (ret)
 279                 goto cpuhp_online_error;
 280
 281 #else   /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 282         {
 283                 int cpu;
 284
 285 #ifdef CONFIG_HOTPLUG_CPU
 286                 perf_field->nb.notifier_call =
 287                         lttng_perf_counter_cpu_hp_callback;
 288                 perf_field->nb.priority = 0;
 289                 register_cpu_notifier(&perf_field->nb);
 290 #endif
 291                 get_online_cpus();
 292                 for_each_online_cpu(cpu) {
 293                         events[cpu] = wrapper_perf_event_create_kernel_counter(attr,
 294                                                 cpu, NULL, overflow_callback);
 295                         if (!events[cpu] || IS_ERR(events[cpu])) {
 296                                 ret = -EINVAL;
 297                                 goto counter_error;
 298                         }
 299                         if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
 300                                 ret = -EBUSY;
 301                                 goto counter_busy;
 302                         }
 303                 }
 304                 put_online_cpus();
 305                 perf_field->hp_enable = 1;
 306         }
 307 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 308
 309         field->destroy = lttng_destroy_perf_counter_field;
 310
 311         field->event_field.name = name_alloc;
 312         field->event_field.type.atype = atype_integer;
 313         field->event_field.type.u.basic.integer.size = sizeof(uint64_t) * CHAR_BIT;
 314         field->event_field.type.u.basic.integer.alignment = lttng_alignof(uint64_t) * CHAR_BIT;
 315         field->event_field.type.u.basic.integer.signedness = lttng_is_signed_type(uint64_t);
 316         field->event_field.type.u.basic.integer.reverse_byte_order = 0;
 317         field->event_field.type.u.basic.integer.base = 10;
 318         field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
 319         field->get_size = perf_counter_get_size;
 320         field->record = perf_counter_record;
 321         field->u.perf_counter = perf_field;
 322         lttng_context_update(*ctx);
 323
 324         wrapper_vmalloc_sync_mappings();
 325         return 0;
 326
 327 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
 328 cpuhp_online_error:
 329         {
 330                 int remove_ret;
 331
 332                 remove_ret = cpuhp_state_remove_instance(lttng_hp_prepare,
 333                                 &perf_field->cpuhp_prepare.node);
 334                 WARN_ON(remove_ret);
 335         }
 336 cpuhp_prepare_error:
 337 #else   /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 338 counter_busy:
 339 counter_error:
 340         {
 341                 int cpu;
 342
 343                 for_each_online_cpu(cpu) {
 344                         if (events[cpu] && !IS_ERR(events[cpu]))
 345                                 perf_event_release_kernel(events[cpu]);
 346                 }
 347                 put_online_cpus();
 348 #ifdef CONFIG_HOTPLUG_CPU
 349                 unregister_cpu_notifier(&perf_field->nb);
 350 #endif
 351         }
 352 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
 353 find_error:
 354         lttng_remove_context_field(ctx, field);
 355 append_context_error:
 356         kfree(name_alloc);
 357 name_alloc_error:
 358         kfree(perf_field);
 359 error_alloc_perf_field:
 360         kfree(attr);
 361 error_attr:
 362         lttng_kvfree(events);
 363         return ret;
 364 }