libmarkers/marker.c

   1 /*
   2  * Copyright (C) 2007 Mathieu Desnoyers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  */
  18 //ust// #include <linux/module.h>
  19 //ust// #include <linux/mutex.h>
  20 //ust// #include <linux/types.h>
  21 #include "jhash.h"
  22 #include "list.h"
  23 #include "rcupdate.h"
  24 //ust// #include <linux/marker.h>
  25 #include <errno.h>
  26 //ust// #include <linux/slab.h>
  27 //ust// #include <linux/immediate.h>
  28 //ust// #include <linux/sched.h>
  29 //ust// #include <linux/uaccess.h>
  30 //ust// #include <linux/user_marker.h>
  31 //ust// #include <linux/ltt-tracer.h>
  32
  33 #include "marker.h"
  34 #include "kernelcompat.h"
  35 #include "usterr.h"
  36 #include "channels.h"
  37 #include "tracercore.h"
  38 #include "tracer.h"
  39
  40 extern struct marker __start___markers[];
  41 extern struct marker __stop___markers[];
  42
  43 /* Set to 1 to enable marker debug output */
  44 static const int marker_debug;
  45
  46 /*
  47  * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
  48  * and module markers and the hash table.
  49  */
  50 static DEFINE_MUTEX(markers_mutex);
  51
  52 void lock_markers(void)
  53 {
  54         mutex_lock(&markers_mutex);
  55 }
  56
  57 void unlock_markers(void)
  58 {
  59         mutex_unlock(&markers_mutex);
  60 }
  61
  62 /*
  63  * Marker hash table, containing the active markers.
  64  * Protected by module_mutex.
  65  */
  66 #define MARKER_HASH_BITS 6
  67 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
  68 static struct hlist_head marker_table[MARKER_TABLE_SIZE];
  69
  70 /*
  71  * Note about RCU :
  72  * It is used to make sure every handler has finished using its private data
  73  * between two consecutive operation (add or remove) on a given marker.  It is
  74  * also used to delay the free of multiple probes array until a quiescent state
  75  * is reached.
  76  * marker entries modifications are protected by the markers_mutex.
  77  */
  78 struct marker_entry {
  79         struct hlist_node hlist;
  80         char *format;
  81         char *name;
  82                         /* Probe wrapper */
  83         void (*call)(const struct marker *mdata, void *call_private, ...);
  84         struct marker_probe_closure single;
  85         struct marker_probe_closure *multi;
  86         int refcount;   /* Number of times armed. 0 if disarmed. */
  87         struct rcu_head rcu;
  88         void *oldptr;
  89         int rcu_pending;
  90         u16 channel_id;
  91         u16 event_id;
  92         unsigned char ptype:1;
  93         unsigned char format_allocated:1;
  94         char channel[0];        /* Contains channel'\0'name'\0'format'\0' */
  95 };
  96
  97 #ifdef CONFIG_MARKERS_USERSPACE
  98 static void marker_update_processes(void);
  99 #else
 100 static void marker_update_processes(void)
 101 {
 102 }
 103 #endif
 104
 105 /**
 106  * __mark_empty_function - Empty probe callback
 107  * @mdata: marker data
 108  * @probe_private: probe private data
 109  * @call_private: call site private data
 110  * @fmt: format string
 111  * @...: variable argument list
 112  *
 113  * Empty callback provided as a probe to the markers. By providing this to a
 114  * disabled marker, we make sure the  execution flow is always valid even
 115  * though the function pointer change and the marker enabling are two distinct
 116  * operations that modifies the execution flow of preemptible code.
 117  */
 118 notrace void __mark_empty_function(const struct marker *mdata,
 119         void *probe_private, void *call_private, const char *fmt, va_list *args)
 120 {
 121 }
 122 //ust// EXPORT_SYMBOL_GPL(__mark_empty_function);
 123
 124 /*
 125  * marker_probe_cb Callback that prepares the variable argument list for probes.
 126  * @mdata: pointer of type struct marker
 127  * @call_private: caller site private data
 128  * @...:  Variable argument list.
 129  *
 130  * Since we do not use "typical" pointer based RCU in the 1 argument case, we
 131  * need to put a full smp_rmb() in this branch. This is why we do not use
 132  * rcu_dereference() for the pointer read.
 133  */
 134 notrace void marker_probe_cb(const struct marker *mdata,
 135                 void *call_private, ...)
 136 {
 137         va_list args;
 138         char ptype;
 139
 140         /*
 141          * rcu_read_lock_sched does two things : disabling preemption to make
 142          * sure the teardown of the callbacks can be done correctly when they
 143          * are in modules and they insure RCU read coherency.
 144          */
 145 //ust// rcu_read_lock_sched_notrace();
 146         ptype = mdata->ptype;
 147         if (likely(!ptype)) {
 148                 marker_probe_func *func;
 149                 /* Must read the ptype before ptr. They are not data dependant,
 150                  * so we put an explicit smp_rmb() here. */
 151                 smp_rmb();
 152                 func = mdata->single.func;
 153                 /* Must read the ptr before private data. They are not data
 154                  * dependant, so we put an explicit smp_rmb() here. */
 155                 smp_rmb();
 156                 va_start(args, call_private);
 157                 func(mdata, mdata->single.probe_private, call_private,
 158                         mdata->format, &args);
 159                 va_end(args);
 160         } else {
 161                 struct marker_probe_closure *multi;
 162                 int i;
 163                 /*
 164                  * Read mdata->ptype before mdata->multi.
 165                  */
 166                 smp_rmb();
 167                 multi = mdata->multi;
 168                 /*
 169                  * multi points to an array, therefore accessing the array
 170                  * depends on reading multi. However, even in this case,
 171                  * we must insure that the pointer is read _before_ the array
 172                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 173                  * in the fast path, so put the explicit barrier here.
 174                  */
 175                 smp_read_barrier_depends();
 176                 for (i = 0; multi[i].func; i++) {
 177                         va_start(args, call_private);
 178                         multi[i].func(mdata, multi[i].probe_private,
 179                                 call_private, mdata->format, &args);
 180                         va_end(args);
 181                 }
 182         }
 183 //ust// rcu_read_unlock_sched_notrace();
 184 }
 185 //ust// EXPORT_SYMBOL_GPL(marker_probe_cb);
 186
 187 /*
 188  * marker_probe_cb Callback that does not prepare the variable argument list.
 189  * @mdata: pointer of type struct marker
 190  * @call_private: caller site private data
 191  * @...:  Variable argument list.
 192  *
 193  * Should be connected to markers "MARK_NOARGS".
 194  */
 195 static notrace void marker_probe_cb_noarg(const struct marker *mdata,
 196                 void *call_private, ...)
 197 {
 198         va_list args;   /* not initialized */
 199         char ptype;
 200
 201 //ust// rcu_read_lock_sched_notrace();
 202         ptype = mdata->ptype;
 203         if (likely(!ptype)) {
 204                 marker_probe_func *func;
 205                 /* Must read the ptype before ptr. They are not data dependant,
 206                  * so we put an explicit smp_rmb() here. */
 207                 smp_rmb();
 208                 func = mdata->single.func;
 209                 /* Must read the ptr before private data. They are not data
 210                  * dependant, so we put an explicit smp_rmb() here. */
 211                 smp_rmb();
 212                 func(mdata, mdata->single.probe_private, call_private,
 213                         mdata->format, &args);
 214         } else {
 215                 struct marker_probe_closure *multi;
 216                 int i;
 217                 /*
 218                  * Read mdata->ptype before mdata->multi.
 219                  */
 220                 smp_rmb();
 221                 multi = mdata->multi;
 222                 /*
 223                  * multi points to an array, therefore accessing the array
 224                  * depends on reading multi. However, even in this case,
 225                  * we must insure that the pointer is read _before_ the array
 226                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 227                  * in the fast path, so put the explicit barrier here.
 228                  */
 229                 smp_read_barrier_depends();
 230                 for (i = 0; multi[i].func; i++)
 231                         multi[i].func(mdata, multi[i].probe_private,
 232                                 call_private, mdata->format, &args);
 233         }
 234 //ust// rcu_read_unlock_sched_notrace();
 235 }
 236
 237 static void free_old_closure(struct rcu_head *head)
 238 {
 239         struct marker_entry *entry = container_of(head,
 240                 struct marker_entry, rcu);
 241         kfree(entry->oldptr);
 242         /* Make sure we free the data before setting the pending flag to 0 */
 243         smp_wmb();
 244         entry->rcu_pending = 0;
 245 }
 246
 247 static void debug_print_probes(struct marker_entry *entry)
 248 {
 249         int i;
 250
 251         if (!marker_debug)
 252                 return;
 253
 254         if (!entry->ptype) {
 255                 printk(KERN_DEBUG "Single probe : %p %p\n",
 256                         entry->single.func,
 257                         entry->single.probe_private);
 258         } else {
 259                 for (i = 0; entry->multi[i].func; i++)
 260                         printk(KERN_DEBUG "Multi probe %d : %p %p\n", i,
 261                                 entry->multi[i].func,
 262                                 entry->multi[i].probe_private);
 263         }
 264 }
 265
 266 static struct marker_probe_closure *
 267 marker_entry_add_probe(struct marker_entry *entry,
 268                 marker_probe_func *probe, void *probe_private)
 269 {
 270         int nr_probes = 0;
 271         struct marker_probe_closure *old, *new;
 272
 273         WARN_ON(!probe);
 274
 275         debug_print_probes(entry);
 276         old = entry->multi;
 277         if (!entry->ptype) {
 278                 if (entry->single.func == probe &&
 279                                 entry->single.probe_private == probe_private)
 280                         return ERR_PTR(-EBUSY);
 281                 if (entry->single.func == __mark_empty_function) {
 282                         /* 0 -> 1 probes */
 283                         entry->single.func = probe;
 284                         entry->single.probe_private = probe_private;
 285                         entry->refcount = 1;
 286                         entry->ptype = 0;
 287                         debug_print_probes(entry);
 288                         return NULL;
 289                 } else {
 290                         /* 1 -> 2 probes */
 291                         nr_probes = 1;
 292                         old = NULL;
 293                 }
 294         } else {
 295                 /* (N -> N+1), (N != 0, 1) probes */
 296                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 297                         if (old[nr_probes].func == probe
 298                                         && old[nr_probes].probe_private
 299                                                 == probe_private)
 300                                 return ERR_PTR(-EBUSY);
 301         }
 302         /* + 2 : one for new probe, one for NULL func */
 303         new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure),
 304                         GFP_KERNEL);
 305         if (new == NULL)
 306                 return ERR_PTR(-ENOMEM);
 307         if (!old)
 308                 new[0] = entry->single;
 309         else
 310                 memcpy(new, old,
 311                         nr_probes * sizeof(struct marker_probe_closure));
 312         new[nr_probes].func = probe;
 313         new[nr_probes].probe_private = probe_private;
 314         entry->refcount = nr_probes + 1;
 315         entry->multi = new;
 316         entry->ptype = 1;
 317         debug_print_probes(entry);
 318         return old;
 319 }
 320
 321 static struct marker_probe_closure *
 322 marker_entry_remove_probe(struct marker_entry *entry,
 323                 marker_probe_func *probe, void *probe_private)
 324 {
 325         int nr_probes = 0, nr_del = 0, i;
 326         struct marker_probe_closure *old, *new;
 327
 328         old = entry->multi;
 329
 330         debug_print_probes(entry);
 331         if (!entry->ptype) {
 332                 /* 0 -> N is an error */
 333                 WARN_ON(entry->single.func == __mark_empty_function);
 334                 /* 1 -> 0 probes */
 335                 WARN_ON(probe && entry->single.func != probe);
 336                 WARN_ON(entry->single.probe_private != probe_private);
 337                 entry->single.func = __mark_empty_function;
 338                 entry->refcount = 0;
 339                 entry->ptype = 0;
 340                 debug_print_probes(entry);
 341                 return NULL;
 342         } else {
 343                 /* (N -> M), (N > 1, M >= 0) probes */
 344                 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 345                         if ((!probe || old[nr_probes].func == probe)
 346                                         && old[nr_probes].probe_private
 347                                                 == probe_private)
 348                                 nr_del++;
 349                 }
 350         }
 351
 352         if (nr_probes - nr_del == 0) {
 353                 /* N -> 0, (N > 1) */
 354                 entry->single.func = __mark_empty_function;
 355                 entry->refcount = 0;
 356                 entry->ptype = 0;
 357         } else if (nr_probes - nr_del == 1) {
 358                 /* N -> 1, (N > 1) */
 359                 for (i = 0; old[i].func; i++)
 360                         if ((probe && old[i].func != probe) ||
 361                                         old[i].probe_private != probe_private)
 362                                 entry->single = old[i];
 363                 entry->refcount = 1;
 364                 entry->ptype = 0;
 365         } else {
 366                 int j = 0;
 367                 /* N -> M, (N > 1, M > 1) */
 368                 /* + 1 for NULL */
 369                 new = kzalloc((nr_probes - nr_del + 1)
 370                         * sizeof(struct marker_probe_closure), GFP_KERNEL);
 371                 if (new == NULL)
 372                         return ERR_PTR(-ENOMEM);
 373                 for (i = 0; old[i].func; i++)
 374                         if ((probe && old[i].func != probe) ||
 375                                         old[i].probe_private != probe_private)
 376                                 new[j++] = old[i];
 377                 entry->refcount = nr_probes - nr_del;
 378                 entry->ptype = 1;
 379                 entry->multi = new;
 380         }
 381         debug_print_probes(entry);
 382         return old;
 383 }
 384
 385 /*
 386  * Get marker if the marker is present in the marker hash table.
 387  * Must be called with markers_mutex held.
 388  * Returns NULL if not present.
 389  */
 390 static struct marker_entry *get_marker(const char *channel, const char *name)
 391 {
 392         struct hlist_head *head;
 393         struct hlist_node *node;
 394         struct marker_entry *e;
 395         size_t channel_len = strlen(channel) + 1;
 396         size_t name_len = strlen(name) + 1;
 397         u32 hash;
 398
 399         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 400         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 401         hlist_for_each_entry(e, node, head, hlist) {
 402                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name))
 403                         return e;
 404         }
 405         return NULL;
 406 }
 407
 408 /*
 409  * Add the marker to the marker hash table. Must be called with markers_mutex
 410  * held.
 411  */
 412 static struct marker_entry *add_marker(const char *channel, const char *name,
 413                 const char *format)
 414 {
 415         struct hlist_head *head;
 416         struct hlist_node *node;
 417         struct marker_entry *e;
 418         size_t channel_len = strlen(channel) + 1;
 419         size_t name_len = strlen(name) + 1;
 420         size_t format_len = 0;
 421         u32 hash;
 422
 423         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 424         if (format)
 425                 format_len = strlen(format) + 1;
 426         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 427         hlist_for_each_entry(e, node, head, hlist) {
 428                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 429                         printk(KERN_NOTICE
 430                                 "Marker %s.%s busy\n", channel, name);
 431                         return ERR_PTR(-EBUSY); /* Already there */
 432                 }
 433         }
 434         /*
 435          * Using kmalloc here to allocate a variable length element. Could
 436          * cause some memory fragmentation if overused.
 437          */
 438         e = kmalloc(sizeof(struct marker_entry)
 439                     + channel_len + name_len + format_len,
 440                     GFP_KERNEL);
 441         if (!e)
 442                 return ERR_PTR(-ENOMEM);
 443         memcpy(e->channel, channel, channel_len);
 444         e->name = &e->channel[channel_len];
 445         memcpy(e->name, name, name_len);
 446         if (format) {
 447                 e->format = &e->name[channel_len + name_len];
 448                 memcpy(e->format, format, format_len);
 449                 if (strcmp(e->format, MARK_NOARGS) == 0)
 450                         e->call = marker_probe_cb_noarg;
 451                 else
 452                         e->call = marker_probe_cb;
 453                 trace_mark(metadata, core_marker_format,
 454                            "channel %s name %s format %s",
 455                            e->channel, e->name, e->format);
 456         } else {
 457                 e->format = NULL;
 458                 e->call = marker_probe_cb;
 459         }
 460         e->single.func = __mark_empty_function;
 461         e->single.probe_private = NULL;
 462         e->multi = NULL;
 463         e->ptype = 0;
 464         e->format_allocated = 0;
 465         e->refcount = 0;
 466         e->rcu_pending = 0;
 467         hlist_add_head(&e->hlist, head);
 468         return e;
 469 }
 470
 471 /*
 472  * Remove the marker from the marker hash table. Must be called with mutex_lock
 473  * held.
 474  */
 475 static int remove_marker(const char *channel, const char *name)
 476 {
 477         struct hlist_head *head;
 478         struct hlist_node *node;
 479         struct marker_entry *e;
 480         int found = 0;
 481         size_t channel_len = strlen(channel) + 1;
 482         size_t name_len = strlen(name) + 1;
 483         u32 hash;
 484         int ret;
 485
 486         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 487         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 488         hlist_for_each_entry(e, node, head, hlist) {
 489                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 490                         found = 1;
 491                         break;
 492                 }
 493         }
 494         if (!found)
 495                 return -ENOENT;
 496         if (e->single.func != __mark_empty_function)
 497                 return -EBUSY;
 498         hlist_del(&e->hlist);
 499         if (e->format_allocated)
 500                 kfree(e->format);
 501         ret = ltt_channels_unregister(e->channel);
 502         WARN_ON(ret);
 503         /* Make sure the call_rcu has been executed */
 504         if (e->rcu_pending)
 505                 rcu_barrier_sched();
 506         kfree(e);
 507         return 0;
 508 }
 509
 510 /*
 511  * Set the mark_entry format to the format found in the element.
 512  */
 513 static int marker_set_format(struct marker_entry *entry, const char *format)
 514 {
 515         entry->format = kstrdup(format, GFP_KERNEL);
 516         if (!entry->format)
 517                 return -ENOMEM;
 518         entry->format_allocated = 1;
 519
 520         trace_mark(metadata, core_marker_format,
 521                    "channel %s name %s format %s",
 522                    entry->channel, entry->name, entry->format);
 523         return 0;
 524 }
 525
 526 /*
 527  * Sets the probe callback corresponding to one marker.
 528  */
 529 static int set_marker(struct marker_entry *entry, struct marker *elem,
 530                 int active)
 531 {
 532         int ret = 0;
 533         WARN_ON(strcmp(entry->name, elem->name) != 0);
 534
 535         if (entry->format) {
 536                 if (strcmp(entry->format, elem->format) != 0) {
 537                         printk(KERN_NOTICE
 538                                 "Format mismatch for probe %s "
 539                                 "(%s), marker (%s)\n",
 540                                 entry->name,
 541                                 entry->format,
 542                                 elem->format);
 543                         return -EPERM;
 544                 }
 545         } else {
 546                 ret = marker_set_format(entry, elem->format);
 547                 if (ret)
 548                         return ret;
 549         }
 550
 551         /*
 552          * probe_cb setup (statically known) is done here. It is
 553          * asynchronous with the rest of execution, therefore we only
 554          * pass from a "safe" callback (with argument) to an "unsafe"
 555          * callback (does not set arguments).
 556          */
 557         elem->call = entry->call;
 558         elem->channel_id = entry->channel_id;
 559         elem->event_id = entry->event_id;
 560         /*
 561          * Sanity check :
 562          * We only update the single probe private data when the ptr is
 563          * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
 564          */
 565         WARN_ON(elem->single.func != __mark_empty_function
 566                 && elem->single.probe_private != entry->single.probe_private
 567                 && !elem->ptype);
 568         elem->single.probe_private = entry->single.probe_private;
 569         /*
 570          * Make sure the private data is valid when we update the
 571          * single probe ptr.
 572          */
 573         smp_wmb();
 574         elem->single.func = entry->single.func;
 575         /*
 576          * We also make sure that the new probe callbacks array is consistent
 577          * before setting a pointer to it.
 578          */
 579         rcu_assign_pointer(elem->multi, entry->multi);
 580         /*
 581          * Update the function or multi probe array pointer before setting the
 582          * ptype.
 583          */
 584         smp_wmb();
 585         elem->ptype = entry->ptype;
 586
 587 //ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) {
 588 //ust//         WARN_ON(!elem->tp_cb);
 589 //ust//         /*
 590 //ust//          * It is ok to directly call the probe registration because type
 591 //ust//          * checking has been done in the __trace_mark_tp() macro.
 592 //ust//          */
 593 //ust//
 594 //ust//         if (active) {
 595 //ust//                 /*
 596 //ust//                  * try_module_get should always succeed because we hold
 597 //ust//                  * markers_mutex to get the tp_cb address.
 598 //ust//                  */
 599 //ust//                 ret = try_module_get(__module_text_address(
 600 //ust//                         (unsigned long)elem->tp_cb));
 601 //ust//                 BUG_ON(!ret);
 602 //ust//                 ret = tracepoint_probe_register_noupdate(
 603 //ust//                         elem->tp_name,
 604 //ust//                         elem->tp_cb);
 605 //ust//         } else {
 606 //ust//                 ret = tracepoint_probe_unregister_noupdate(
 607 //ust//                         elem->tp_name,
 608 //ust//                         elem->tp_cb);
 609 //ust//                 /*
 610 //ust//                  * tracepoint_probe_update_all() must be called
 611 //ust//                  * before the module containing tp_cb is unloaded.
 612 //ust//                  */
 613 //ust//                 module_put(__module_text_address(
 614 //ust//                         (unsigned long)elem->tp_cb));
 615 //ust//         }
 616 //ust// }
 617         elem->state__imv = active;
 618
 619         return ret;
 620 }
 621
 622 /*
 623  * Disable a marker and its probe callback.
 624  * Note: only waiting an RCU period after setting elem->call to the empty
 625  * function insures that the original callback is not used anymore. This insured
 626  * by rcu_read_lock_sched around the call site.
 627  */
 628 static void disable_marker(struct marker *elem)
 629 {
 630         int ret;
 631
 632         /* leave "call" as is. It is known statically. */
 633 //ust// if (elem->tp_name && _imv_read(elem->state)) {
 634 //ust//         WARN_ON(!elem->tp_cb);
 635 //ust//         /*
 636 //ust//          * It is ok to directly call the probe registration because type
 637 //ust//          * checking has been done in the __trace_mark_tp() macro.
 638 //ust//          */
 639 //ust//         ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
 640 //ust//                 elem->tp_cb);
 641 //ust//         WARN_ON(ret);
 642 //ust//         /*
 643 //ust//          * tracepoint_probe_update_all() must be called
 644 //ust//          * before the module containing tp_cb is unloaded.
 645 //ust//          */
 646 //ust//         module_put(__module_text_address((unsigned long)elem->tp_cb));
 647 //ust// }
 648         elem->state__imv = 0;
 649         elem->single.func = __mark_empty_function;
 650         /* Update the function before setting the ptype */
 651         smp_wmb();
 652         elem->ptype = 0;        /* single probe */
 653         /*
 654          * Leave the private data and channel_id/event_id there, because removal
 655          * is racy and should be done only after an RCU period. These are never
 656          * used until the next initialization anyway.
 657          */
 658 }
 659
 660 /**
 661  * marker_update_probe_range - Update a probe range
 662  * @begin: beginning of the range
 663  * @end: end of the range
 664  *
 665  * Updates the probe callback corresponding to a range of markers.
 666  */
 667 void marker_update_probe_range(struct marker *begin,
 668         struct marker *end)
 669 {
 670         struct marker *iter;
 671         struct marker_entry *mark_entry;
 672
 673         mutex_lock(&markers_mutex);
 674         for (iter = begin; iter < end; iter++) {
 675                 mark_entry = get_marker(iter->channel, iter->name);
 676                 if (mark_entry) {
 677                         set_marker(mark_entry, iter, !!mark_entry->refcount);
 678                         /*
 679                          * ignore error, continue
 680                          */
 681                 } else {
 682                         disable_marker(iter);
 683                 }
 684         }
 685         mutex_unlock(&markers_mutex);
 686 }
 687
 688 /*
 689  * Update probes, removing the faulty probes.
 690  *
 691  * Internal callback only changed before the first probe is connected to it.
 692  * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
 693  * transitions.  All other transitions will leave the old private data valid.
 694  * This makes the non-atomicity of the callback/private data updates valid.
 695  *
 696  * "special case" updates :
 697  * 0 -> 1 callback
 698  * 1 -> 0 callback
 699  * 1 -> 2 callbacks
 700  * 2 -> 1 callbacks
 701  * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
 702  * Site effect : marker_set_format may delete the marker entry (creating a
 703  * replacement).
 704  */
 705 static void marker_update_probes(void)
 706 {
 707         /* Core kernel markers */
 708         marker_update_probe_range(__start___markers, __stop___markers);
 709         /* Markers in modules. */
 710 //ust// module_update_markers();
 711 //ust// tracepoint_probe_update_all();
 712         /* Update immediate values */
 713         core_imv_update();
 714 //ust// module_imv_update();
 715         marker_update_processes();
 716 }
 717
 718 /**
 719  * marker_probe_register -  Connect a probe to a marker
 720  * @channel: marker channel
 721  * @name: marker name
 722  * @format: format string
 723  * @probe: probe handler
 724  * @probe_private: probe private data
 725  *
 726  * private data must be a valid allocated memory address, or NULL.
 727  * Returns 0 if ok, error value on error.
 728  * The probe address must at least be aligned on the architecture pointer size.
 729  */
 730 int marker_probe_register(const char *channel, const char *name,
 731                           const char *format, marker_probe_func *probe,
 732                           void *probe_private)
 733 {
 734         struct marker_entry *entry;
 735         int ret = 0, ret_err;
 736         struct marker_probe_closure *old;
 737         int first_probe = 0;
 738
 739         mutex_lock(&markers_mutex);
 740         entry = get_marker(channel, name);
 741         if (!entry) {
 742                 first_probe = 1;
 743                 entry = add_marker(channel, name, format);
 744                 if (IS_ERR(entry))
 745                         ret = PTR_ERR(entry);
 746                 if (ret)
 747                         goto end;
 748                 ret = ltt_channels_register(channel);
 749                 if (ret)
 750                         goto error_remove_marker;
 751                 ret = ltt_channels_get_index_from_name(channel);
 752                 if (ret < 0)
 753                         goto error_unregister_channel;
 754                 entry->channel_id = ret;
 755                 ret = ltt_channels_get_event_id(channel, name);
 756                 if (ret < 0)
 757                         goto error_unregister_channel;
 758                 entry->event_id = ret;
 759                 ret = 0;
 760                 trace_mark(metadata, core_marker_id,
 761                            "channel %s name %s event_id %hu "
 762                            "int #1u%zu long #1u%zu pointer #1u%zu "
 763                            "size_t #1u%zu alignment #1u%u",
 764                            channel, name, entry->event_id,
 765                            sizeof(int), sizeof(long), sizeof(void *),
 766                            sizeof(size_t), ltt_get_alignment());
 767         } else if (format) {
 768                 if (!entry->format)
 769                         ret = marker_set_format(entry, format);
 770                 else if (strcmp(entry->format, format))
 771                         ret = -EPERM;
 772                 if (ret)
 773                         goto end;
 774         }
 775
 776         /*
 777          * If we detect that a call_rcu is pending for this marker,
 778          * make sure it's executed now.
 779          */
 780         if (entry->rcu_pending)
 781                 rcu_barrier_sched();
 782         old = marker_entry_add_probe(entry, probe, probe_private);
 783         if (IS_ERR(old)) {
 784                 ret = PTR_ERR(old);
 785                 if (first_probe)
 786                         goto error_unregister_channel;
 787                 else
 788                         goto end;
 789         }
 790         mutex_unlock(&markers_mutex);
 791
 792         marker_update_probes();
 793
 794         mutex_lock(&markers_mutex);
 795         entry = get_marker(channel, name);
 796         if (!entry)
 797                 goto end;
 798         if (entry->rcu_pending)
 799                 rcu_barrier_sched();
 800         entry->oldptr = old;
 801         entry->rcu_pending = 1;
 802         /* write rcu_pending before calling the RCU callback */
 803         smp_wmb();
 804         call_rcu_sched(&entry->rcu, free_old_closure);
 805         goto end;
 806
 807 error_unregister_channel:
 808         ret_err = ltt_channels_unregister(channel);
 809         WARN_ON(ret_err);
 810 error_remove_marker:
 811         ret_err = remove_marker(channel, name);
 812         WARN_ON(ret_err);
 813 end:
 814         mutex_unlock(&markers_mutex);
 815         return ret;
 816 }
 817 //ust// EXPORT_SYMBOL_GPL(marker_probe_register);
 818
 819 /**
 820  * marker_probe_unregister -  Disconnect a probe from a marker
 821  * @channel: marker channel
 822  * @name: marker name
 823  * @probe: probe function pointer
 824  * @probe_private: probe private data
 825  *
 826  * Returns the private data given to marker_probe_register, or an ERR_PTR().
 827  * We do not need to call a synchronize_sched to make sure the probes have
 828  * finished running before doing a module unload, because the module unload
 829  * itself uses stop_machine(), which insures that every preempt disabled section
 830  * have finished.
 831  */
 832 int marker_probe_unregister(const char *channel, const char *name,
 833                             marker_probe_func *probe, void *probe_private)
 834 {
 835         struct marker_entry *entry;
 836         struct marker_probe_closure *old;
 837         int ret = -ENOENT;
 838
 839         mutex_lock(&markers_mutex);
 840         entry = get_marker(channel, name);
 841         if (!entry)
 842                 goto end;
 843         if (entry->rcu_pending)
 844                 rcu_barrier_sched();
 845         old = marker_entry_remove_probe(entry, probe, probe_private);
 846         mutex_unlock(&markers_mutex);
 847
 848         marker_update_probes();
 849
 850         mutex_lock(&markers_mutex);
 851         entry = get_marker(channel, name);
 852         if (!entry)
 853                 goto end;
 854         if (entry->rcu_pending)
 855                 rcu_barrier_sched();
 856         entry->oldptr = old;
 857         entry->rcu_pending = 1;
 858         /* write rcu_pending before calling the RCU callback */
 859         smp_wmb();
 860         call_rcu_sched(&entry->rcu, free_old_closure);
 861         remove_marker(channel, name);   /* Ignore busy error message */
 862         ret = 0;
 863 end:
 864         mutex_unlock(&markers_mutex);
 865         return ret;
 866 }
 867 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister);
 868
 869 static struct marker_entry *
 870 get_marker_from_private_data(marker_probe_func *probe, void *probe_private)
 871 {
 872         struct marker_entry *entry;
 873         unsigned int i;
 874         struct hlist_head *head;
 875         struct hlist_node *node;
 876
 877         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
 878                 head = &marker_table[i];
 879                 hlist_for_each_entry(entry, node, head, hlist) {
 880                         if (!entry->ptype) {
 881                                 if (entry->single.func == probe
 882                                                 && entry->single.probe_private
 883                                                 == probe_private)
 884                                         return entry;
 885                         } else {
 886                                 struct marker_probe_closure *closure;
 887                                 closure = entry->multi;
 888                                 for (i = 0; closure[i].func; i++) {
 889                                         if (closure[i].func == probe &&
 890                                                         closure[i].probe_private
 891                                                         == probe_private)
 892                                                 return entry;
 893                                 }
 894                         }
 895                 }
 896         }
 897         return NULL;
 898 }
 899
 900 /**
 901  * marker_probe_unregister_private_data -  Disconnect a probe from a marker
 902  * @probe: probe function
 903  * @probe_private: probe private data
 904  *
 905  * Unregister a probe by providing the registered private data.
 906  * Only removes the first marker found in hash table.
 907  * Return 0 on success or error value.
 908  * We do not need to call a synchronize_sched to make sure the probes have
 909  * finished running before doing a module unload, because the module unload
 910  * itself uses stop_machine(), which insures that every preempt disabled section
 911  * have finished.
 912  */
 913 int marker_probe_unregister_private_data(marker_probe_func *probe,
 914                 void *probe_private)
 915 {
 916         struct marker_entry *entry;
 917         int ret = 0;
 918         struct marker_probe_closure *old;
 919         const char *channel = NULL, *name = NULL;
 920
 921         mutex_lock(&markers_mutex);
 922         entry = get_marker_from_private_data(probe, probe_private);
 923         if (!entry) {
 924                 ret = -ENOENT;
 925                 goto end;
 926         }
 927         if (entry->rcu_pending)
 928                 rcu_barrier_sched();
 929         old = marker_entry_remove_probe(entry, NULL, probe_private);
 930         channel = kstrdup(entry->channel, GFP_KERNEL);
 931         name = kstrdup(entry->name, GFP_KERNEL);
 932         mutex_unlock(&markers_mutex);
 933
 934         marker_update_probes();
 935
 936         mutex_lock(&markers_mutex);
 937         entry = get_marker(channel, name);
 938         if (!entry)
 939                 goto end;
 940         if (entry->rcu_pending)
 941                 rcu_barrier_sched();
 942         entry->oldptr = old;
 943         entry->rcu_pending = 1;
 944         /* write rcu_pending before calling the RCU callback */
 945         smp_wmb();
 946         call_rcu_sched(&entry->rcu, free_old_closure);
 947         /* Ignore busy error message */
 948         remove_marker(channel, name);
 949 end:
 950         mutex_unlock(&markers_mutex);
 951         kfree(channel);
 952         kfree(name);
 953         return ret;
 954 }
 955 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
 956
 957 /**
 958  * marker_get_private_data - Get a marker's probe private data
 959  * @channel: marker channel
 960  * @name: marker name
 961  * @probe: probe to match
 962  * @num: get the nth matching probe's private data
 963  *
 964  * Returns the nth private data pointer (starting from 0) matching, or an
 965  * ERR_PTR.
 966  * Returns the private data pointer, or an ERR_PTR.
 967  * The private data pointer should _only_ be dereferenced if the caller is the
 968  * owner of the data, or its content could vanish. This is mostly used to
 969  * confirm that a caller is the owner of a registered probe.
 970  */
 971 void *marker_get_private_data(const char *channel, const char *name,
 972                               marker_probe_func *probe, int num)
 973 {
 974         struct hlist_head *head;
 975         struct hlist_node *node;
 976         struct marker_entry *e;
 977         size_t channel_len = strlen(channel) + 1;
 978         size_t name_len = strlen(name) + 1;
 979         int i;
 980         u32 hash;
 981
 982         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 983         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 984         hlist_for_each_entry(e, node, head, hlist) {
 985                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 986                         if (!e->ptype) {
 987                                 if (num == 0 && e->single.func == probe)
 988                                         return e->single.probe_private;
 989                         } else {
 990                                 struct marker_probe_closure *closure;
 991                                 int match = 0;
 992                                 closure = e->multi;
 993                                 for (i = 0; closure[i].func; i++) {
 994                                         if (closure[i].func != probe)
 995                                                 continue;
 996                                         if (match++ == num)
 997                                                 return closure[i].probe_private;
 998                                 }
 999                         }
1000                         break;
1001                 }
1002         }
1003         return ERR_PTR(-ENOENT);
1004 }
1005 //ust// EXPORT_SYMBOL_GPL(marker_get_private_data);
1006
1007 /**
1008  * markers_compact_event_ids - Compact markers event IDs and reassign channels
1009  *
1010  * Called when no channel users are active by the channel infrastructure.
1011  * Called with lock_markers() and channel mutex held.
1012  */
1013 //ust// void markers_compact_event_ids(void)
1014 //ust// {
1015 //ust//         struct marker_entry *entry;
1016 //ust//         unsigned int i;
1017 //ust//         struct hlist_head *head;
1018 //ust//         struct hlist_node *node;
1019 //ust//         int ret;
1020 //ust//
1021 //ust//         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
1022 //ust//                 head = &marker_table[i];
1023 //ust//                 hlist_for_each_entry(entry, node, head, hlist) {
1024 //ust//                         ret = ltt_channels_get_index_from_name(entry->channel);
1025 //ust//                         WARN_ON(ret < 0);
1026 //ust//                         entry->channel_id = ret;
1027 //ust//                         ret = _ltt_channels_get_event_id(entry->channel,
1028 //ust//                                                          entry->name);
1029 //ust//                         WARN_ON(ret < 0);
1030 //ust//                         entry->event_id = ret;
1031 //ust//                 }
1032 //ust//         }
1033 //ust// }
1034
1035 //ust//#ifdef CONFIG_MODULES
1036
1037 /**
1038  * marker_get_iter_range - Get a next marker iterator given a range.
1039  * @marker: current markers (in), next marker (out)
1040  * @begin: beginning of the range
1041  * @end: end of the range
1042  *
1043  * Returns whether a next marker has been found (1) or not (0).
1044  * Will return the first marker in the range if the input marker is NULL.
1045  */
1046 int marker_get_iter_range(struct marker **marker, struct marker *begin,
1047         struct marker *end)
1048 {
1049         if (!*marker && begin != end) {
1050                 *marker = begin;
1051                 return 1;
1052         }
1053         if (*marker >= begin && *marker < end)
1054                 return 1;
1055         return 0;
1056 }
1057 //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range);
1058
1059 static void marker_get_iter(struct marker_iter *iter)
1060 {
1061         int found = 0;
1062
1063         /* Core kernel markers */
1064         if (!iter->lib) {
1065                 found = marker_get_iter_range(&iter->marker,
1066                                 __start___markers, __stop___markers);
1067                 if (found)
1068                         goto end;
1069         }
1070         /* Markers in modules. */
1071         found = lib_get_iter_markers(iter);
1072 end:
1073         if (!found)
1074                 marker_iter_reset(iter);
1075 }
1076
1077 void marker_iter_start(struct marker_iter *iter)
1078 {
1079         marker_get_iter(iter);
1080 }
1081 //ust// EXPORT_SYMBOL_GPL(marker_iter_start);
1082
1083 void marker_iter_next(struct marker_iter *iter)
1084 {
1085         iter->marker++;
1086         /*
1087          * iter->marker may be invalid because we blindly incremented it.
1088          * Make sure it is valid by marshalling on the markers, getting the
1089          * markers from following modules if necessary.
1090          */
1091         marker_get_iter(iter);
1092 }
1093 //ust// EXPORT_SYMBOL_GPL(marker_iter_next);
1094
1095 void marker_iter_stop(struct marker_iter *iter)
1096 {
1097 }
1098 //ust// EXPORT_SYMBOL_GPL(marker_iter_stop);
1099
1100 void marker_iter_reset(struct marker_iter *iter)
1101 {
1102         iter->lib = NULL;
1103         iter->marker = NULL;
1104 }
1105 //ust// EXPORT_SYMBOL_GPL(marker_iter_reset);
1106
1107 #ifdef CONFIG_MARKERS_USERSPACE
1108 /*
1109  * must be called with current->user_markers_mutex held
1110  */
1111 static void free_user_marker(char __user *state, struct hlist_head *head)
1112 {
1113         struct user_marker *umark;
1114         struct hlist_node *pos, *n;
1115
1116         hlist_for_each_entry_safe(umark, pos, n, head, hlist) {
1117                 if (umark->state == state) {
1118                         hlist_del(&umark->hlist);
1119                         kfree(umark);
1120                 }
1121         }
1122 }
1123
1124 //ust// asmlinkage long sys_marker(char __user *name, char __user *format,
1125 //ust//                 char __user *state, int reg)
1126 //ust// {
1127 //ust//         struct user_marker *umark;
1128 //ust//         long len;
1129 //ust//         struct marker_entry *entry;
1130 //ust//         int ret = 0;
1131 //ust//
1132 //ust//         printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n",
1133 //ust//                 current->comm, reg ? "registers" : "unregisters",
1134 //ust//                 name, state);
1135 //ust//         if (reg) {
1136 //ust//                 umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL);
1137 //ust//                 umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0';
1138 //ust//                 umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0';
1139 //ust//                 umark->state = state;
1140 //ust//                 len = strncpy_from_user(umark->name, name,
1141 //ust//                         MAX_USER_MARKER_NAME_LEN - 1);
1142 //ust//                 if (len < 0) {
1143 //ust//                         ret = -EFAULT;
1144 //ust//                         goto error;
1145 //ust//                 }
1146 //ust//                 len = strncpy_from_user(umark->format, format,
1147 //ust//                         MAX_USER_MARKER_FORMAT_LEN - 1);
1148 //ust//                 if (len < 0) {
1149 //ust//                         ret = -EFAULT;
1150 //ust//                         goto error;
1151 //ust//                 }
1152 //ust//                 printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name,
1153 //ust//                         umark->format);
1154 //ust//                 mutex_lock(&markers_mutex);
1155 //ust//                 entry = get_marker("userspace", umark->name);
1156 //ust//                 if (entry) {
1157 //ust//                         if (entry->format &&
1158 //ust//                                 strcmp(entry->format, umark->format) != 0) {
1159 //ust//                                 printk(" error, wrong format in process %s",
1160 //ust//                                         current->comm);
1161 //ust//                                 ret = -EPERM;
1162 //ust//                                 goto error_unlock;
1163 //ust//                         }
1164 //ust//                         printk(" %s", !!entry->refcount
1165 //ust//                                         ? "enabled" : "disabled");
1166 //ust//                         if (put_user(!!entry->refcount, state)) {
1167 //ust//                                 ret = -EFAULT;
1168 //ust//                                 goto error_unlock;
1169 //ust//                         }
1170 //ust//                         printk("\n");
1171 //ust//                 } else {
1172 //ust//                         printk(" disabled\n");
1173 //ust//                         if (put_user(0, umark->state)) {
1174 //ust//                                 printk(KERN_WARNING
1175 //ust//                                         "Marker in %s caused a fault\n",
1176 //ust//                                         current->comm);
1177 //ust//                                 goto error_unlock;
1178 //ust//                         }
1179 //ust//                 }
1180 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1181 //ust//                 hlist_add_head(&umark->hlist,
1182 //ust//                         &current->group_leader->user_markers);
1183 //ust//                 current->group_leader->user_markers_sequence++;
1184 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1185 //ust//                 mutex_unlock(&markers_mutex);
1186 //ust//         } else {
1187 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1188 //ust//                 free_user_marker(state,
1189 //ust//                         &current->group_leader->user_markers);
1190 //ust//                 current->group_leader->user_markers_sequence++;
1191 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1192 //ust//         }
1193 //ust//         goto end;
1194 //ust// error_unlock:
1195 //ust//         mutex_unlock(&markers_mutex);
1196 //ust// error:
1197 //ust//         kfree(umark);
1198 //ust// end:
1199 //ust//         return ret;
1200 //ust// }
1201 //ust//
1202 //ust// /*
1203 //ust//  * Types :
1204 //ust//  * string : 0
1205 //ust//  */
1206 //ust// asmlinkage long sys_trace(int type, uint16_t id,
1207 //ust//                 char __user *ubuf)
1208 //ust// {
1209 //ust//         long ret = -EPERM;
1210 //ust//         char *page;
1211 //ust//         int len;
1212 //ust//
1213 //ust//         switch (type) {
1214 //ust//         case 0: /* String */
1215 //ust//                 ret = -ENOMEM;
1216 //ust//                 page = (char *)__get_free_page(GFP_TEMPORARY);
1217 //ust//                 if (!page)
1218 //ust//                         goto string_out;
1219 //ust//                 len = strncpy_from_user(page, ubuf, PAGE_SIZE);
1220 //ust//                 if (len < 0) {
1221 //ust//                         ret = -EFAULT;
1222 //ust//                         goto string_err;
1223 //ust//                 }
1224 //ust//                 trace_mark(userspace, string, "string %s", page);
1225 //ust// string_err:
1226 //ust//                 free_page((unsigned long) page);
1227 //ust// string_out:
1228 //ust//                 break;
1229 //ust//         default:
1230 //ust//                 break;
1231 //ust//         }
1232 //ust//         return ret;
1233 //ust// }
1234
1235 //ust// static void marker_update_processes(void)
1236 //ust// {
1237 //ust//         struct task_struct *g, *t;
1238 //ust//
1239 //ust//         /*
1240 //ust//          * markers_mutex is taken to protect the p->user_markers read.
1241 //ust//          */
1242 //ust//         mutex_lock(&markers_mutex);
1243 //ust//         read_lock(&tasklist_lock);
1244 //ust//         for_each_process(g) {
1245 //ust//                 WARN_ON(!thread_group_leader(g));
1246 //ust//                 if (hlist_empty(&g->user_markers))
1247 //ust//                         continue;
1248 //ust//                 if (strcmp(g->comm, "testprog") == 0)
1249 //ust//                         printk(KERN_DEBUG "set update pending for testprog\n");
1250 //ust//                 t = g;
1251 //ust//                 do {
1252 //ust//                         /* TODO : implement this thread flag in each arch. */
1253 //ust//                         set_tsk_thread_flag(t, TIF_MARKER_PENDING);
1254 //ust//                 } while ((t = next_thread(t)) != g);
1255 //ust//         }
1256 //ust//         read_unlock(&tasklist_lock);
1257 //ust//         mutex_unlock(&markers_mutex);
1258 //ust// }
1259
1260 /*
1261  * Update current process.
1262  * Note that we have to wait a whole scheduler period before we are sure that
1263  * every running userspace threads have their markers updated.
1264  * (synchronize_sched() can be used to insure this).
1265  */
1266 void marker_update_process(void)
1267 {
1268         struct user_marker *umark;
1269         struct hlist_node *pos;
1270         struct marker_entry *entry;
1271
1272         mutex_lock(&markers_mutex);
1273         mutex_lock(&current->group_leader->user_markers_mutex);
1274         if (strcmp(current->comm, "testprog") == 0)
1275                 printk(KERN_DEBUG "do update pending for testprog\n");
1276         hlist_for_each_entry(umark, pos,
1277                         &current->group_leader->user_markers, hlist) {
1278                 printk(KERN_DEBUG "Updating marker %s in %s\n",
1279                         umark->name, current->comm);
1280                 entry = get_marker("userspace", umark->name);
1281                 if (entry) {
1282                         if (entry->format &&
1283                                 strcmp(entry->format, umark->format) != 0) {
1284                                 printk(KERN_WARNING
1285                                         " error, wrong format in process %s\n",
1286                                         current->comm);
1287                                 break;
1288                         }
1289                         if (put_user(!!entry->refcount, umark->state)) {
1290                                 printk(KERN_WARNING
1291                                         "Marker in %s caused a fault\n",
1292                                         current->comm);
1293                                 break;
1294                         }
1295                 } else {
1296                         if (put_user(0, umark->state)) {
1297                                 printk(KERN_WARNING
1298                                         "Marker in %s caused a fault\n",
1299                                         current->comm);
1300                                 break;
1301                         }
1302                 }
1303         }
1304         clear_thread_flag(TIF_MARKER_PENDING);
1305         mutex_unlock(&current->group_leader->user_markers_mutex);
1306         mutex_unlock(&markers_mutex);
1307 }
1308
1309 /*
1310  * Called at process exit and upon do_execve().
1311  * We assume that when the leader exits, no more references can be done to the
1312  * leader structure by the other threads.
1313  */
1314 void exit_user_markers(struct task_struct *p)
1315 {
1316         struct user_marker *umark;
1317         struct hlist_node *pos, *n;
1318
1319         if (thread_group_leader(p)) {
1320                 mutex_lock(&markers_mutex);
1321                 mutex_lock(&p->user_markers_mutex);
1322                 hlist_for_each_entry_safe(umark, pos, n, &p->user_markers,
1323                         hlist)
1324                     kfree(umark);
1325                 INIT_HLIST_HEAD(&p->user_markers);
1326                 p->user_markers_sequence++;
1327                 mutex_unlock(&p->user_markers_mutex);
1328                 mutex_unlock(&markers_mutex);
1329         }
1330 }
1331
1332 int is_marker_enabled(const char *channel, const char *name)
1333 {
1334         struct marker_entry *entry;
1335
1336         mutex_lock(&markers_mutex);
1337         entry = get_marker(channel, name);
1338         mutex_unlock(&markers_mutex);
1339
1340         return entry && !!entry->refcount;
1341 }
1342 //ust// #endif
1343
1344 int marker_module_notify(struct notifier_block *self,
1345                          unsigned long val, void *data)
1346 {
1347         struct module *mod = data;
1348
1349         switch (val) {
1350         case MODULE_STATE_COMING:
1351                 marker_update_probe_range(mod->markers,
1352                         mod->markers + mod->num_markers);
1353                 break;
1354         case MODULE_STATE_GOING:
1355                 marker_update_probe_range(mod->markers,
1356                         mod->markers + mod->num_markers);
1357                 break;
1358         }
1359         return 0;
1360 }
1361
1362 struct notifier_block marker_module_nb = {
1363         .notifier_call = marker_module_notify,
1364         .priority = 0,
1365 };
1366
1367 //ust// static int init_markers(void)
1368 //ust// {
1369 //ust//         return register_module_notifier(&marker_module_nb);
1370 //ust// }
1371 //ust// __initcall(init_markers);
1372 /* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */
1373
1374 #endif /* CONFIG_MODULES */
1375
1376 void ltt_dump_marker_state(struct ltt_trace_struct *trace)
1377 {
1378         struct marker_iter iter;
1379         struct ltt_probe_private_data call_data;
1380         const char *channel;
1381
1382         call_data.trace = trace;
1383         call_data.serializer = NULL;
1384
1385         marker_iter_reset(&iter);
1386         marker_iter_start(&iter);
1387         for (; iter.marker != NULL; marker_iter_next(&iter)) {
1388                 if (!_imv_read(iter.marker->state))
1389                         continue;
1390                 channel = ltt_channels_get_name_from_index(
1391                                 iter.marker->channel_id);
1392                 __trace_mark(0, metadata, core_marker_id,
1393                         &call_data,
1394                         "channel %s name %s event_id %hu "
1395                         "int #1u%zu long #1u%zu pointer #1u%zu "
1396                         "size_t #1u%zu alignment #1u%u",
1397                         channel,
1398                         iter.marker->name,
1399                         iter.marker->event_id,
1400                         sizeof(int), sizeof(long),
1401                         sizeof(void *), sizeof(size_t),
1402                         ltt_get_alignment());
1403                 if (iter.marker->format)
1404                         __trace_mark(0, metadata,
1405                                 core_marker_format,
1406                                 &call_data,
1407                                 "channel %s name %s format %s",
1408                                 channel,
1409                                 iter.marker->name,
1410                                 iter.marker->format);
1411         }
1412         marker_iter_stop(&iter);
1413 }
1414 //ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state);
1415
1416
1417 static LIST_HEAD(libs);
1418
1419 /*
1420  * Returns 0 if current not found.
1421  * Returns 1 if current found.
1422  */
1423 int lib_get_iter_markers(struct marker_iter *iter)
1424 {
1425         struct lib *iter_lib;
1426         int found = 0;
1427
1428 //ust// mutex_lock(&module_mutex);
1429         list_for_each_entry(iter_lib, &libs, list) {
1430                 if (iter_lib < iter->lib)
1431                         continue;
1432                 else if (iter_lib > iter->lib)
1433                         iter->marker = NULL;
1434                 found = marker_get_iter_range(&iter->marker,
1435                         iter_lib->markers_start,
1436                         iter_lib->markers_start + iter_lib->markers_count);
1437                 if (found) {
1438                         iter->lib = iter_lib;
1439                         break;
1440                 }
1441         }
1442 //ust// mutex_unlock(&module_mutex);
1443         return found;
1444 }
1445
1446 int marker_register_lib(struct marker *markers_start, int markers_count)
1447 {
1448         struct lib *pl;
1449
1450         pl = (struct lib *) malloc(sizeof(struct lib));
1451
1452         pl->markers_start = markers_start;
1453         pl->markers_count = markers_count;
1454
1455         list_add(&pl->list, &libs);
1456
1457         printf("just registered a markers section from %p and having %d markers\n", markers_start, markers_count);
1458
1459         return 0;
1460 }