libust/tracepoint.c

   1 /*
   2  * Copyright (C) 2008-2011 Mathieu Desnoyers
   3  * Copyright (C) 2009 Pierre-Marc Fournier
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation;
   8  * version 2.1 of the License.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  *
  19  * Ported to userspace by Pierre-Marc Fournier.
  20  */
  21
  22 #define _LGPL_SOURCE
  23 #include <errno.h>
  24 #include <ust/tracepoint.h>
  25 #include <ust/tracepoint-internal.h>
  26 #include <ust/core.h>
  27 #include <ust/kcompat/kcompat.h>
  28 #include <urcu-bp.h>
  29 #include <urcu/hlist.h>
  30 #include <urcu/uatomic.h>
  31
  32 #include <ust/usterr-signal-safe.h>
  33
  34 /* Set to 1 to enable tracepoint debug output */
  35 static const int tracepoint_debug;
  36 static int initialized;
  37 static void (*new_tracepoint_cb)(struct tracepoint *);
  38
  39 /* libraries that contain tracepoints (struct tracepoint_lib) */
  40 static CDS_LIST_HEAD(libs);
  41
  42 /*
  43  * The UST lock protects the library tracepoints, the hash table, and
  44  * the library list.
  45  * All calls to the tracepoint API must be protected by the UST lock,
  46  * excepts calls to tracepoint_register_lib and
  47  * tracepoint_unregister_lib, which take the UST lock themselves.
  48  */
  49
  50 /*
  51  * Tracepoint hash table, containing the active tracepoints.
  52  * Protected by tracepoints_mutex.
  53  */
  54 #define TRACEPOINT_HASH_BITS 6
  55 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
  56 static struct cds_hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
  57
  58 static CDS_LIST_HEAD(old_probes);
  59 static int need_update;
  60
  61 /*
  62  * Note about RCU :
  63  * It is used to to delay the free of multiple probes array until a quiescent
  64  * state is reached.
  65  * Tracepoint entries modifications are protected by the tracepoints_mutex.
  66  */
  67 struct tracepoint_entry {
  68         struct cds_hlist_node hlist;
  69         struct tracepoint_probe *probes;
  70         int refcount;   /* Number of times armed. 0 if disarmed. */
  71         char name[0];
  72 };
  73
  74 struct tp_probes {
  75         union {
  76                 struct cds_list_head list;
  77         } u;
  78         struct tracepoint_probe probes[0];
  79 };
  80
  81 static inline void *allocate_probes(int count)
  82 {
  83         struct tp_probes *p  = zmalloc(count * sizeof(struct tracepoint_probe)
  84                         + sizeof(struct tp_probes));
  85         return p == NULL ? NULL : p->probes;
  86 }
  87
  88 static inline void release_probes(void *old)
  89 {
  90         if (old) {
  91                 struct tp_probes *tp_probes = _ust_container_of(old,
  92                         struct tp_probes, probes[0]);
  93                 synchronize_rcu();
  94                 free(tp_probes);
  95         }
  96 }
  97
  98 static void debug_print_probes(struct tracepoint_entry *entry)
  99 {
 100         int i;
 101
 102         if (!tracepoint_debug || !entry->probes)
 103                 return;
 104
 105         for (i = 0; entry->probes[i].func; i++)
 106                 DBG("Probe %d : %p", i, entry->probes[i].func);
 107 }
 108
 109 static void *
 110 tracepoint_entry_add_probe(struct tracepoint_entry *entry,
 111                            void *probe, void *data)
 112 {
 113         int nr_probes = 0;
 114         struct tracepoint_probe *old, *new;
 115
 116         WARN_ON(!probe);
 117
 118         debug_print_probes(entry);
 119         old = entry->probes;
 120         if (old) {
 121                 /* (N -> N+1), (N != 0, 1) probes */
 122                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 123                         if (old[nr_probes].func == probe &&
 124                             old[nr_probes].data == data)
 125                                 return ERR_PTR(-EEXIST);
 126         }
 127         /* + 2 : one for new probe, one for NULL func */
 128         new = allocate_probes(nr_probes + 2);
 129         if (new == NULL)
 130                 return ERR_PTR(-ENOMEM);
 131         if (old)
 132                 memcpy(new, old, nr_probes * sizeof(struct tracepoint_probe));
 133         new[nr_probes].func = probe;
 134         new[nr_probes].data = data;
 135         new[nr_probes + 1].func = NULL;
 136         entry->refcount = nr_probes + 1;
 137         entry->probes = new;
 138         debug_print_probes(entry);
 139         return old;
 140 }
 141
 142 static void *
 143 tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe,
 144                               void *data)
 145 {
 146         int nr_probes = 0, nr_del = 0, i;
 147         struct tracepoint_probe *old, *new;
 148
 149         old = entry->probes;
 150
 151         if (!old)
 152                 return ERR_PTR(-ENOENT);
 153
 154         debug_print_probes(entry);
 155         /* (N -> M), (N > 1, M >= 0) probes */
 156         for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 157                 if (!probe ||
 158                      (old[nr_probes].func == probe &&
 159                      old[nr_probes].data == data))
 160                         nr_del++;
 161         }
 162
 163         if (nr_probes - nr_del == 0) {
 164                 /* N -> 0, (N > 1) */
 165                 entry->probes = NULL;
 166                 entry->refcount = 0;
 167                 debug_print_probes(entry);
 168                 return old;
 169         } else {
 170                 int j = 0;
 171                 /* N -> M, (N > 1, M > 0) */
 172                 /* + 1 for NULL */
 173                 new = allocate_probes(nr_probes - nr_del + 1);
 174                 if (new == NULL)
 175                         return ERR_PTR(-ENOMEM);
 176                 for (i = 0; old[i].func; i++)
 177                         if (probe &&
 178                             (old[i].func != probe || old[i].data != data))
 179                                 new[j++] = old[i];
 180                 new[nr_probes - nr_del].func = NULL;
 181                 entry->refcount = nr_probes - nr_del;
 182                 entry->probes = new;
 183         }
 184         debug_print_probes(entry);
 185         return old;
 186 }
 187
 188 /*
 189  * Get tracepoint if the tracepoint is present in the tracepoint hash table.
 190  * Must be called with tracepoints_mutex held.
 191  * Returns NULL if not present.
 192  */
 193 static struct tracepoint_entry *get_tracepoint(const char *name)
 194 {
 195         struct cds_hlist_head *head;
 196         struct cds_hlist_node *node;
 197         struct tracepoint_entry *e;
 198         u32 hash = jhash(name, strlen(name), 0);
 199
 200         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 201         cds_hlist_for_each_entry(e, node, head, hlist) {
 202                 if (!strcmp(name, e->name))
 203                         return e;
 204         }
 205         return NULL;
 206 }
 207
 208 /*
 209  * Add the tracepoint to the tracepoint hash table. Must be called with
 210  * tracepoints_mutex held.
 211  */
 212 static struct tracepoint_entry *add_tracepoint(const char *name)
 213 {
 214         struct cds_hlist_head *head;
 215         struct cds_hlist_node *node;
 216         struct tracepoint_entry *e;
 217         size_t name_len = strlen(name) + 1;
 218         u32 hash = jhash(name, name_len-1, 0);
 219
 220         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 221         cds_hlist_for_each_entry(e, node, head, hlist) {
 222                 if (!strcmp(name, e->name)) {
 223                         DBG("tracepoint %s busy", name);
 224                         return ERR_PTR(-EEXIST);        /* Already there */
 225                 }
 226         }
 227         /*
 228          * Using zmalloc here to allocate a variable length element. Could
 229          * cause some memory fragmentation if overused.
 230          */
 231         e = zmalloc(sizeof(struct tracepoint_entry) + name_len);
 232         if (!e)
 233                 return ERR_PTR(-ENOMEM);
 234         memcpy(&e->name[0], name, name_len);
 235         e->probes = NULL;
 236         e->refcount = 0;
 237         cds_hlist_add_head(&e->hlist, head);
 238         return e;
 239 }
 240
 241 /*
 242  * Remove the tracepoint from the tracepoint hash table. Must be called with
 243  * ust_lock held.
 244  */
 245 static inline void remove_tracepoint(struct tracepoint_entry *e)
 246 {
 247         cds_hlist_del(&e->hlist);
 248         free(e);
 249 }
 250
 251 /*
 252  * Sets the probe callback corresponding to one tracepoint.
 253  */
 254 static void set_tracepoint(struct tracepoint_entry **entry,
 255         struct tracepoint *elem, int active)
 256 {
 257         WARN_ON(strcmp((*entry)->name, elem->name) != 0);
 258
 259         /*
 260          * rcu_assign_pointer has a cmm_smp_wmb() which makes sure that the new
 261          * probe callbacks array is consistent before setting a pointer to it.
 262          * This array is referenced by __DO_TRACE from
 263          * include/linux/tracepoints.h. A matching cmm_smp_read_barrier_depends()
 264          * is used.
 265          */
 266         rcu_assign_pointer(elem->probes, (*entry)->probes);
 267         elem->state = active;
 268 }
 269
 270 /*
 271  * Disable a tracepoint and its probe callback.
 272  * Note: only waiting an RCU period after setting elem->call to the empty
 273  * function insures that the original callback is not used anymore. This insured
 274  * by preempt_disable around the call site.
 275  */
 276 static void disable_tracepoint(struct tracepoint *elem)
 277 {
 278         elem->state = 0;
 279         rcu_assign_pointer(elem->probes, NULL);
 280 }
 281
 282 /**
 283  * tracepoint_update_probe_range - Update a probe range
 284  * @begin: beginning of the range
 285  * @end: end of the range
 286  *
 287  * Updates the probe callback corresponding to a range of tracepoints.
 288  */
 289 static
 290 void tracepoint_update_probe_range(struct tracepoint * const *begin,
 291                                    struct tracepoint * const *end)
 292 {
 293         struct tracepoint * const *iter;
 294         struct tracepoint_entry *mark_entry;
 295
 296         for (iter = begin; iter < end; iter++) {
 297                 if (!*iter)
 298                         continue;       /* skip dummy */
 299                 if (!(*iter)->name) {
 300                         disable_tracepoint(*iter);
 301                         continue;
 302                 }
 303                 mark_entry = get_tracepoint((*iter)->name);
 304                 if (mark_entry) {
 305                         set_tracepoint(&mark_entry, *iter,
 306                                         !!mark_entry->refcount);
 307                 } else {
 308                         disable_tracepoint(*iter);
 309                 }
 310         }
 311 }
 312
 313 static void lib_update_tracepoints(void)
 314 {
 315         struct tracepoint_lib *lib;
 316
 317         cds_list_for_each_entry(lib, &libs, list) {
 318                 tracepoint_update_probe_range(lib->tracepoints_start,
 319                                 lib->tracepoints_start + lib->tracepoints_count);
 320         }
 321 }
 322
 323 /*
 324  * Update probes, removing the faulty probes.
 325  */
 326 static void tracepoint_update_probes(void)
 327 {
 328         /* tracepoints registered from libraries and executable. */
 329         lib_update_tracepoints();
 330 }
 331
 332 static struct tracepoint_probe *
 333 tracepoint_add_probe(const char *name, void *probe, void *data)
 334 {
 335         struct tracepoint_entry *entry;
 336         struct tracepoint_probe *old;
 337
 338         entry = get_tracepoint(name);
 339         if (!entry) {
 340                 entry = add_tracepoint(name);
 341                 if (IS_ERR(entry))
 342                         return (struct tracepoint_probe *)entry;
 343         }
 344         old = tracepoint_entry_add_probe(entry, probe, data);
 345         if (IS_ERR(old) && !entry->refcount)
 346                 remove_tracepoint(entry);
 347         return old;
 348 }
 349
 350 /**
 351  * __tracepoint_probe_register -  Connect a probe to a tracepoint
 352  * @name: tracepoint name
 353  * @probe: probe handler
 354  *
 355  * Returns 0 if ok, error value on error.
 356  * The probe address must at least be aligned on the architecture pointer size.
 357  * Called with the UST lock held.
 358  */
 359 int __tracepoint_probe_register(const char *name, void *probe, void *data)
 360 {
 361         void *old;
 362
 363         old = tracepoint_add_probe(name, probe, data);
 364         if (IS_ERR(old))
 365                 return PTR_ERR(old);
 366
 367         tracepoint_update_probes();             /* may update entry */
 368         release_probes(old);
 369         return 0;
 370 }
 371
 372 static void *tracepoint_remove_probe(const char *name, void *probe, void *data)
 373 {
 374         struct tracepoint_entry *entry;
 375         void *old;
 376
 377         entry = get_tracepoint(name);
 378         if (!entry)
 379                 return ERR_PTR(-ENOENT);
 380         old = tracepoint_entry_remove_probe(entry, probe, data);
 381         if (IS_ERR(old))
 382                 return old;
 383         if (!entry->refcount)
 384                 remove_tracepoint(entry);
 385         return old;
 386 }
 387
 388 /**
 389  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 390  * @name: tracepoint name
 391  * @probe: probe function pointer
 392  * @probe: probe data pointer
 393  *
 394  * Called with the UST lock held.
 395  */
 396 int __tracepoint_probe_unregister(const char *name, void *probe, void *data)
 397 {
 398         void *old;
 399
 400         old = tracepoint_remove_probe(name, probe, data);
 401         if (IS_ERR(old))
 402                 return PTR_ERR(old);
 403
 404         tracepoint_update_probes();             /* may update entry */
 405         release_probes(old);
 406         return 0;
 407 }
 408
 409 static void tracepoint_add_old_probes(void *old)
 410 {
 411         need_update = 1;
 412         if (old) {
 413                 struct tp_probes *tp_probes = _ust_container_of(old,
 414                         struct tp_probes, probes[0]);
 415                 cds_list_add(&tp_probes->u.list, &old_probes);
 416         }
 417 }
 418
 419 /**
 420  * tracepoint_probe_register_noupdate -  register a probe but not connect
 421  * @name: tracepoint name
 422  * @probe: probe handler
 423  *
 424  * caller must call tracepoint_probe_update_all()
 425  * Called with the UST lock held.
 426  */
 427 int tracepoint_probe_register_noupdate(const char *name, void *probe,
 428                                        void *data)
 429 {
 430         void *old;
 431
 432         old = tracepoint_add_probe(name, probe, data);
 433         if (IS_ERR(old)) {
 434                 return PTR_ERR(old);
 435         }
 436         tracepoint_add_old_probes(old);
 437         return 0;
 438 }
 439
 440 /**
 441  * tracepoint_probe_unregister_noupdate -  remove a probe but not disconnect
 442  * @name: tracepoint name
 443  * @probe: probe function pointer
 444  *
 445  * caller must call tracepoint_probe_update_all()
 446  * Called with the UST lock held.
 447  */
 448 int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
 449                                          void *data)
 450 {
 451         void *old;
 452
 453         old = tracepoint_remove_probe(name, probe, data);
 454         if (IS_ERR(old)) {
 455                 return PTR_ERR(old);
 456         }
 457         tracepoint_add_old_probes(old);
 458         return 0;
 459 }
 460
 461 /**
 462  * tracepoint_probe_update_all -  update tracepoints
 463  * Called with the UST lock held.
 464  */
 465 void tracepoint_probe_update_all(void)
 466 {
 467         CDS_LIST_HEAD(release_probes);
 468         struct tp_probes *pos, *next;
 469
 470         if (!need_update) {
 471                 return;
 472         }
 473         if (!cds_list_empty(&old_probes))
 474                 cds_list_replace_init(&old_probes, &release_probes);
 475         need_update = 0;
 476
 477         tracepoint_update_probes();
 478         cds_list_for_each_entry_safe(pos, next, &release_probes, u.list) {
 479                 cds_list_del(&pos->u.list);
 480                 synchronize_rcu();
 481                 free(pos);
 482         }
 483 }
 484
 485 /*
 486  * Returns 0 if current not found.
 487  * Returns 1 if current found.
 488  *
 489  * Called with tracepoint mutex held
 490  */
 491 int lib_get_iter_tracepoints(struct tracepoint_iter *iter)
 492 {
 493         struct tracepoint_lib *iter_lib;
 494         int found = 0;
 495
 496         cds_list_for_each_entry(iter_lib, &libs, list) {
 497                 if (iter_lib < iter->lib)
 498                         continue;
 499                 else if (iter_lib > iter->lib)
 500                         iter->tracepoint = NULL;
 501                 found = tracepoint_get_iter_range(&iter->tracepoint,
 502                         iter_lib->tracepoints_start,
 503                         iter_lib->tracepoints_start + iter_lib->tracepoints_count);
 504                 if (found) {
 505                         iter->lib = iter_lib;
 506                         break;
 507                 }
 508         }
 509         return found;
 510 }
 511
 512 /**
 513  * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
 514  * @tracepoint: current tracepoints (in), next tracepoint (out)
 515  * @begin: beginning of the range
 516  * @end: end of the range
 517  *
 518  * Returns whether a next tracepoint has been found (1) or not (0).
 519  * Will return the first tracepoint in the range if the input tracepoint is
 520  * NULL.
 521  * Called with tracepoint mutex held.
 522  */
 523 int tracepoint_get_iter_range(struct tracepoint * const **tracepoint,
 524         struct tracepoint * const *begin, struct tracepoint * const *end)
 525 {
 526         if (!*tracepoint && begin != end)
 527                 *tracepoint = begin;
 528         while (*tracepoint >= begin && *tracepoint < end) {
 529                 if (!**tracepoint)
 530                         (*tracepoint)++;        /* skip dummy */
 531                 else
 532                         return 1;
 533         }
 534         return 0;
 535 }
 536
 537 /*
 538  * Called with tracepoint mutex held.
 539  */
 540 static void tracepoint_get_iter(struct tracepoint_iter *iter)
 541 {
 542         int found = 0;
 543
 544         /* tracepoints in libs. */
 545         found = lib_get_iter_tracepoints(iter);
 546         if (!found)
 547                 tracepoint_iter_reset(iter);
 548 }
 549
 550 /*
 551  * Called with UST lock held.
 552  */
 553 void tracepoint_iter_start(struct tracepoint_iter *iter)
 554 {
 555         tracepoint_get_iter(iter);
 556 }
 557
 558 /*
 559  * Called with UST lock held.
 560  */
 561 void tracepoint_iter_next(struct tracepoint_iter *iter)
 562 {
 563         iter->tracepoint++;
 564         /*
 565          * iter->tracepoint may be invalid because we blindly incremented it.
 566          * Make sure it is valid by marshalling on the tracepoints, getting the
 567          * tracepoints from following modules if necessary.
 568          */
 569         tracepoint_get_iter(iter);
 570 }
 571
 572 /*
 573  * Called with UST lock held.
 574  */
 575 void tracepoint_iter_stop(struct tracepoint_iter *iter)
 576 {
 577 }
 578
 579 void tracepoint_iter_reset(struct tracepoint_iter *iter)
 580 {
 581         iter->tracepoint = NULL;
 582 }
 583
 584 void tracepoint_set_new_tracepoint_cb(void (*cb)(struct tracepoint *))
 585 {
 586         new_tracepoint_cb = cb;
 587 }
 588
 589 static void new_tracepoints(struct tracepoint * const *start, struct tracepoint * const *end)
 590 {
 591         if (new_tracepoint_cb) {
 592                 struct tracepoint * const *t;
 593
 594                 for (t = start; t < end; t++) {
 595                         if (*t)
 596                                 new_tracepoint_cb(*t);
 597                 }
 598         }
 599 }
 600
 601 int tracepoint_register_lib(struct tracepoint * const *tracepoints_start,
 602                             int tracepoints_count)
 603 {
 604         struct tracepoint_lib *pl, *iter;
 605
 606         init_tracepoint();
 607
 608         pl = (struct tracepoint_lib *) zmalloc(sizeof(struct tracepoint_lib));
 609
 610         pl->tracepoints_start = tracepoints_start;
 611         pl->tracepoints_count = tracepoints_count;
 612
 613         ust_lock();
 614         /*
 615          * We sort the libs by struct lib pointer address.
 616          */
 617         cds_list_for_each_entry_reverse(iter, &libs, list) {
 618                 BUG_ON(iter == pl);    /* Should never be in the list twice */
 619                 if (iter < pl) {
 620                         /* We belong to the location right after iter. */
 621                         cds_list_add(&pl->list, &iter->list);
 622                         goto lib_added;
 623                 }
 624         }
 625         /* We should be added at the head of the list */
 626         cds_list_add(&pl->list, &libs);
 627 lib_added:
 628         new_tracepoints(tracepoints_start, tracepoints_start + tracepoints_count);
 629
 630         /* TODO: update just the loaded lib */
 631         lib_update_tracepoints();
 632         ust_unlock();
 633
 634         DBG("just registered a tracepoints section from %p and having %d tracepoints",
 635                 tracepoints_start, tracepoints_count);
 636
 637         return 0;
 638 }
 639
 640 int tracepoint_unregister_lib(struct tracepoint * const *tracepoints_start)
 641 {
 642         struct tracepoint_lib *lib;
 643
 644         ust_lock();
 645         cds_list_for_each_entry(lib, &libs, list) {
 646                 if (lib->tracepoints_start == tracepoints_start) {
 647                         struct tracepoint_lib *lib2free = lib;
 648                         cds_list_del(&lib->list);
 649                         free(lib2free);
 650                         break;
 651                 }
 652         }
 653         ust_unlock();
 654
 655         return 0;
 656 }
 657
 658 void init_tracepoint(void)
 659 {
 660         if (uatomic_xchg(&initialized, 1) == 1)
 661                 return;
 662         init_usterr();
 663 }
 664
 665 void exit_tracepoint(void)
 666 {
 667         initialized = 0;
 668 }