libmarkers/marker.c

   1 /*
   2  * Copyright (C) 2007 Mathieu Desnoyers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  */
  18 //ust// #include <linux/module.h>
  19 //ust// #include <linux/mutex.h>
  20 //ust// #include <linux/types.h>
  21 #include "jhash.h"
  22 #include "list.h"
  23 #include "rcupdate.h"
  24 //ust// #include <linux/marker.h>
  25 #include <errno.h>
  26 //ust// #include <linux/slab.h>
  27 //ust// #include <linux/immediate.h>
  28 //ust// #include <linux/sched.h>
  29 //ust// #include <linux/uaccess.h>
  30 //ust// #include <linux/user_marker.h>
  31 //ust// #include <linux/ltt-tracer.h>
  32
  33 #include "marker.h"
  34 #include "kernelcompat.h"
  35 #include "usterr.h"
  36 #include "channels.h"
  37 #include "tracercore.h"
  38 #include "tracer.h"
  39
  40 extern struct marker __start___markers[] __attribute__((visibility("hidden")));
  41 extern struct marker __stop___markers[] __attribute__((visibility("hidden")));
  42
  43 /* Set to 1 to enable marker debug output */
  44 static const int marker_debug;
  45
  46 /*
  47  * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
  48  * and module markers and the hash table.
  49  */
  50 static DEFINE_MUTEX(markers_mutex);
  51
  52 void lock_markers(void)
  53 {
  54         mutex_lock(&markers_mutex);
  55 }
  56
  57 void unlock_markers(void)
  58 {
  59         mutex_unlock(&markers_mutex);
  60 }
  61
  62 /*
  63  * Marker hash table, containing the active markers.
  64  * Protected by module_mutex.
  65  */
  66 #define MARKER_HASH_BITS 6
  67 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
  68 static struct hlist_head marker_table[MARKER_TABLE_SIZE];
  69
  70 /*
  71  * Note about RCU :
  72  * It is used to make sure every handler has finished using its private data
  73  * between two consecutive operation (add or remove) on a given marker.  It is
  74  * also used to delay the free of multiple probes array until a quiescent state
  75  * is reached.
  76  * marker entries modifications are protected by the markers_mutex.
  77  */
  78 struct marker_entry {
  79         struct hlist_node hlist;
  80         char *format;
  81         char *name;
  82                         /* Probe wrapper */
  83         void (*call)(const struct marker *mdata, void *call_private, ...);
  84         struct marker_probe_closure single;
  85         struct marker_probe_closure *multi;
  86         int refcount;   /* Number of times armed. 0 if disarmed. */
  87         struct rcu_head rcu;
  88         void *oldptr;
  89         int rcu_pending;
  90         u16 channel_id;
  91         u16 event_id;
  92         unsigned char ptype:1;
  93         unsigned char format_allocated:1;
  94         char channel[0];        /* Contains channel'\0'name'\0'format'\0' */
  95 };
  96
  97 #ifdef CONFIG_MARKERS_USERSPACE
  98 static void marker_update_processes(void);
  99 #else
 100 static void marker_update_processes(void)
 101 {
 102 }
 103 #endif
 104
 105 /**
 106  * __mark_empty_function - Empty probe callback
 107  * @mdata: marker data
 108  * @probe_private: probe private data
 109  * @call_private: call site private data
 110  * @fmt: format string
 111  * @...: variable argument list
 112  *
 113  * Empty callback provided as a probe to the markers. By providing this to a
 114  * disabled marker, we make sure the  execution flow is always valid even
 115  * though the function pointer change and the marker enabling are two distinct
 116  * operations that modifies the execution flow of preemptible code.
 117  */
 118 notrace void __mark_empty_function(const struct marker *mdata,
 119         void *probe_private, void *call_private, const char *fmt, va_list *args)
 120 {
 121 }
 122 //ust// EXPORT_SYMBOL_GPL(__mark_empty_function);
 123
 124 /*
 125  * marker_probe_cb Callback that prepares the variable argument list for probes.
 126  * @mdata: pointer of type struct marker
 127  * @call_private: caller site private data
 128  * @...:  Variable argument list.
 129  *
 130  * Since we do not use "typical" pointer based RCU in the 1 argument case, we
 131  * need to put a full smp_rmb() in this branch. This is why we do not use
 132  * rcu_dereference() for the pointer read.
 133  */
 134 notrace void marker_probe_cb(const struct marker *mdata,
 135                 void *call_private, ...)
 136 {
 137         va_list args;
 138         char ptype;
 139
 140         /*
 141          * rcu_read_lock_sched does two things : disabling preemption to make
 142          * sure the teardown of the callbacks can be done correctly when they
 143          * are in modules and they insure RCU read coherency.
 144          */
 145 //ust// rcu_read_lock_sched_notrace();
 146         ptype = mdata->ptype;
 147         if (likely(!ptype)) {
 148                 marker_probe_func *func;
 149                 /* Must read the ptype before ptr. They are not data dependant,
 150                  * so we put an explicit smp_rmb() here. */
 151                 smp_rmb();
 152                 func = mdata->single.func;
 153                 /* Must read the ptr before private data. They are not data
 154                  * dependant, so we put an explicit smp_rmb() here. */
 155                 smp_rmb();
 156                 va_start(args, call_private);
 157                 func(mdata, mdata->single.probe_private, call_private,
 158                         mdata->format, &args);
 159                 va_end(args);
 160         } else {
 161                 struct marker_probe_closure *multi;
 162                 int i;
 163                 /*
 164                  * Read mdata->ptype before mdata->multi.
 165                  */
 166                 smp_rmb();
 167                 multi = mdata->multi;
 168                 /*
 169                  * multi points to an array, therefore accessing the array
 170                  * depends on reading multi. However, even in this case,
 171                  * we must insure that the pointer is read _before_ the array
 172                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 173                  * in the fast path, so put the explicit barrier here.
 174                  */
 175                 smp_read_barrier_depends();
 176                 for (i = 0; multi[i].func; i++) {
 177                         va_start(args, call_private);
 178                         multi[i].func(mdata, multi[i].probe_private,
 179                                 call_private, mdata->format, &args);
 180                         va_end(args);
 181                 }
 182         }
 183 //ust// rcu_read_unlock_sched_notrace();
 184 }
 185 //ust// EXPORT_SYMBOL_GPL(marker_probe_cb);
 186
 187 /*
 188  * marker_probe_cb Callback that does not prepare the variable argument list.
 189  * @mdata: pointer of type struct marker
 190  * @call_private: caller site private data
 191  * @...:  Variable argument list.
 192  *
 193  * Should be connected to markers "MARK_NOARGS".
 194  */
 195 static notrace void marker_probe_cb_noarg(const struct marker *mdata,
 196                 void *call_private, ...)
 197 {
 198         va_list args;   /* not initialized */
 199         char ptype;
 200
 201 //ust// rcu_read_lock_sched_notrace();
 202         ptype = mdata->ptype;
 203         if (likely(!ptype)) {
 204                 marker_probe_func *func;
 205                 /* Must read the ptype before ptr. They are not data dependant,
 206                  * so we put an explicit smp_rmb() here. */
 207                 smp_rmb();
 208                 func = mdata->single.func;
 209                 /* Must read the ptr before private data. They are not data
 210                  * dependant, so we put an explicit smp_rmb() here. */
 211                 smp_rmb();
 212                 func(mdata, mdata->single.probe_private, call_private,
 213                         mdata->format, &args);
 214         } else {
 215                 struct marker_probe_closure *multi;
 216                 int i;
 217                 /*
 218                  * Read mdata->ptype before mdata->multi.
 219                  */
 220                 smp_rmb();
 221                 multi = mdata->multi;
 222                 /*
 223                  * multi points to an array, therefore accessing the array
 224                  * depends on reading multi. However, even in this case,
 225                  * we must insure that the pointer is read _before_ the array
 226                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 227                  * in the fast path, so put the explicit barrier here.
 228                  */
 229                 smp_read_barrier_depends();
 230                 for (i = 0; multi[i].func; i++)
 231                         multi[i].func(mdata, multi[i].probe_private,
 232                                 call_private, mdata->format, &args);
 233         }
 234 //ust// rcu_read_unlock_sched_notrace();
 235 }
 236
 237 static void free_old_closure(struct rcu_head *head)
 238 {
 239         struct marker_entry *entry = container_of(head,
 240                 struct marker_entry, rcu);
 241         kfree(entry->oldptr);
 242         /* Make sure we free the data before setting the pending flag to 0 */
 243         smp_wmb();
 244         entry->rcu_pending = 0;
 245 }
 246
 247 static void debug_print_probes(struct marker_entry *entry)
 248 {
 249         int i;
 250
 251         if (!marker_debug)
 252                 return;
 253
 254         if (!entry->ptype) {
 255                 printk(KERN_DEBUG "Single probe : %p %p\n",
 256                         entry->single.func,
 257                         entry->single.probe_private);
 258         } else {
 259                 for (i = 0; entry->multi[i].func; i++)
 260                         printk(KERN_DEBUG "Multi probe %d : %p %p\n", i,
 261                                 entry->multi[i].func,
 262                                 entry->multi[i].probe_private);
 263         }
 264 }
 265
 266 static struct marker_probe_closure *
 267 marker_entry_add_probe(struct marker_entry *entry,
 268                 marker_probe_func *probe, void *probe_private)
 269 {
 270         int nr_probes = 0;
 271         struct marker_probe_closure *old, *new;
 272
 273         WARN_ON(!probe);
 274
 275         debug_print_probes(entry);
 276         old = entry->multi;
 277         if (!entry->ptype) {
 278                 if (entry->single.func == probe &&
 279                                 entry->single.probe_private == probe_private)
 280                         return ERR_PTR(-EBUSY);
 281                 if (entry->single.func == __mark_empty_function) {
 282                         /* 0 -> 1 probes */
 283                         entry->single.func = probe;
 284                         entry->single.probe_private = probe_private;
 285                         entry->refcount = 1;
 286                         entry->ptype = 0;
 287                         debug_print_probes(entry);
 288                         return NULL;
 289                 } else {
 290                         /* 1 -> 2 probes */
 291                         nr_probes = 1;
 292                         old = NULL;
 293                 }
 294         } else {
 295                 /* (N -> N+1), (N != 0, 1) probes */
 296                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 297                         if (old[nr_probes].func == probe
 298                                         && old[nr_probes].probe_private
 299                                                 == probe_private)
 300                                 return ERR_PTR(-EBUSY);
 301         }
 302         /* + 2 : one for new probe, one for NULL func */
 303         new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure),
 304                         GFP_KERNEL);
 305         if (new == NULL)
 306                 return ERR_PTR(-ENOMEM);
 307         if (!old)
 308                 new[0] = entry->single;
 309         else
 310                 memcpy(new, old,
 311                         nr_probes * sizeof(struct marker_probe_closure));
 312         new[nr_probes].func = probe;
 313         new[nr_probes].probe_private = probe_private;
 314         entry->refcount = nr_probes + 1;
 315         entry->multi = new;
 316         entry->ptype = 1;
 317         debug_print_probes(entry);
 318         return old;
 319 }
 320
 321 static struct marker_probe_closure *
 322 marker_entry_remove_probe(struct marker_entry *entry,
 323                 marker_probe_func *probe, void *probe_private)
 324 {
 325         int nr_probes = 0, nr_del = 0, i;
 326         struct marker_probe_closure *old, *new;
 327
 328         old = entry->multi;
 329
 330         debug_print_probes(entry);
 331         if (!entry->ptype) {
 332                 /* 0 -> N is an error */
 333                 WARN_ON(entry->single.func == __mark_empty_function);
 334                 /* 1 -> 0 probes */
 335                 WARN_ON(probe && entry->single.func != probe);
 336                 WARN_ON(entry->single.probe_private != probe_private);
 337                 entry->single.func = __mark_empty_function;
 338                 entry->refcount = 0;
 339                 entry->ptype = 0;
 340                 debug_print_probes(entry);
 341                 return NULL;
 342         } else {
 343                 /* (N -> M), (N > 1, M >= 0) probes */
 344                 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 345                         if ((!probe || old[nr_probes].func == probe)
 346                                         && old[nr_probes].probe_private
 347                                                 == probe_private)
 348                                 nr_del++;
 349                 }
 350         }
 351
 352         if (nr_probes - nr_del == 0) {
 353                 /* N -> 0, (N > 1) */
 354                 entry->single.func = __mark_empty_function;
 355                 entry->refcount = 0;
 356                 entry->ptype = 0;
 357         } else if (nr_probes - nr_del == 1) {
 358                 /* N -> 1, (N > 1) */
 359                 for (i = 0; old[i].func; i++)
 360                         if ((probe && old[i].func != probe) ||
 361                                         old[i].probe_private != probe_private)
 362                                 entry->single = old[i];
 363                 entry->refcount = 1;
 364                 entry->ptype = 0;
 365         } else {
 366                 int j = 0;
 367                 /* N -> M, (N > 1, M > 1) */
 368                 /* + 1 for NULL */
 369                 new = kzalloc((nr_probes - nr_del + 1)
 370                         * sizeof(struct marker_probe_closure), GFP_KERNEL);
 371                 if (new == NULL)
 372                         return ERR_PTR(-ENOMEM);
 373                 for (i = 0; old[i].func; i++)
 374                         if ((probe && old[i].func != probe) ||
 375                                         old[i].probe_private != probe_private)
 376                                 new[j++] = old[i];
 377                 entry->refcount = nr_probes - nr_del;
 378                 entry->ptype = 1;
 379                 entry->multi = new;
 380         }
 381         debug_print_probes(entry);
 382         return old;
 383 }
 384
 385 /*
 386  * Get marker if the marker is present in the marker hash table.
 387  * Must be called with markers_mutex held.
 388  * Returns NULL if not present.
 389  */
 390 static struct marker_entry *get_marker(const char *channel, const char *name)
 391 {
 392         struct hlist_head *head;
 393         struct hlist_node *node;
 394         struct marker_entry *e;
 395         size_t channel_len = strlen(channel) + 1;
 396         size_t name_len = strlen(name) + 1;
 397         u32 hash;
 398
 399         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 400         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 401         hlist_for_each_entry(e, node, head, hlist) {
 402                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name))
 403                         return e;
 404         }
 405         return NULL;
 406 }
 407
 408 /*
 409  * Add the marker to the marker hash table. Must be called with markers_mutex
 410  * held.
 411  */
 412 static struct marker_entry *add_marker(const char *channel, const char *name,
 413                 const char *format)
 414 {
 415         struct hlist_head *head;
 416         struct hlist_node *node;
 417         struct marker_entry *e;
 418         size_t channel_len = strlen(channel) + 1;
 419         size_t name_len = strlen(name) + 1;
 420         size_t format_len = 0;
 421         u32 hash;
 422
 423         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 424         if (format)
 425                 format_len = strlen(format) + 1;
 426         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 427         hlist_for_each_entry(e, node, head, hlist) {
 428                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 429                         printk(KERN_NOTICE
 430                                 "Marker %s.%s busy\n", channel, name);
 431                         return ERR_PTR(-EBUSY); /* Already there */
 432                 }
 433         }
 434         /*
 435          * Using kmalloc here to allocate a variable length element. Could
 436          * cause some memory fragmentation if overused.
 437          */
 438         e = kmalloc(sizeof(struct marker_entry)
 439                     + channel_len + name_len + format_len,
 440                     GFP_KERNEL);
 441         if (!e)
 442                 return ERR_PTR(-ENOMEM);
 443         memcpy(e->channel, channel, channel_len);
 444         e->name = &e->channel[channel_len];
 445         memcpy(e->name, name, name_len);
 446         if (format) {
 447                 e->format = &e->name[channel_len + name_len];
 448                 memcpy(e->format, format, format_len);
 449                 if (strcmp(e->format, MARK_NOARGS) == 0)
 450                         e->call = marker_probe_cb_noarg;
 451                 else
 452                         e->call = marker_probe_cb;
 453                 trace_mark(metadata, core_marker_format,
 454                            "channel %s name %s format %s",
 455                            e->channel, e->name, e->format);
 456         } else {
 457                 e->format = NULL;
 458                 e->call = marker_probe_cb;
 459         }
 460         e->single.func = __mark_empty_function;
 461         e->single.probe_private = NULL;
 462         e->multi = NULL;
 463         e->ptype = 0;
 464         e->format_allocated = 0;
 465         e->refcount = 0;
 466         e->rcu_pending = 0;
 467         hlist_add_head(&e->hlist, head);
 468         return e;
 469 }
 470
 471 /*
 472  * Remove the marker from the marker hash table. Must be called with mutex_lock
 473  * held.
 474  */
 475 static int remove_marker(const char *channel, const char *name)
 476 {
 477         struct hlist_head *head;
 478         struct hlist_node *node;
 479         struct marker_entry *e;
 480         int found = 0;
 481         size_t channel_len = strlen(channel) + 1;
 482         size_t name_len = strlen(name) + 1;
 483         u32 hash;
 484         int ret;
 485
 486         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 487         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 488         hlist_for_each_entry(e, node, head, hlist) {
 489                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 490                         found = 1;
 491                         break;
 492                 }
 493         }
 494         if (!found)
 495                 return -ENOENT;
 496         if (e->single.func != __mark_empty_function)
 497                 return -EBUSY;
 498         hlist_del(&e->hlist);
 499         if (e->format_allocated)
 500                 kfree(e->format);
 501         ret = ltt_channels_unregister(e->channel);
 502         WARN_ON(ret);
 503         /* Make sure the call_rcu has been executed */
 504         if (e->rcu_pending)
 505                 rcu_barrier_sched();
 506         kfree(e);
 507         return 0;
 508 }
 509
 510 /*
 511  * Set the mark_entry format to the format found in the element.
 512  */
 513 static int marker_set_format(struct marker_entry *entry, const char *format)
 514 {
 515         entry->format = kstrdup(format, GFP_KERNEL);
 516         if (!entry->format)
 517                 return -ENOMEM;
 518         entry->format_allocated = 1;
 519
 520         trace_mark(metadata, core_marker_format,
 521                    "channel %s name %s format %s",
 522                    entry->channel, entry->name, entry->format);
 523         return 0;
 524 }
 525
 526 /*
 527  * Sets the probe callback corresponding to one marker.
 528  */
 529 static int set_marker(struct marker_entry *entry, struct marker *elem,
 530                 int active)
 531 {
 532         int ret = 0;
 533         WARN_ON(strcmp(entry->name, elem->name) != 0);
 534
 535         if (entry->format) {
 536                 if (strcmp(entry->format, elem->format) != 0) {
 537                         printk(KERN_NOTICE
 538                                 "Format mismatch for probe %s "
 539                                 "(%s), marker (%s)\n",
 540                                 entry->name,
 541                                 entry->format,
 542                                 elem->format);
 543                         return -EPERM;
 544                 }
 545         } else {
 546                 ret = marker_set_format(entry, elem->format);
 547                 if (ret)
 548                         return ret;
 549         }
 550
 551         /*
 552          * probe_cb setup (statically known) is done here. It is
 553          * asynchronous with the rest of execution, therefore we only
 554          * pass from a "safe" callback (with argument) to an "unsafe"
 555          * callback (does not set arguments).
 556          */
 557         elem->call = entry->call;
 558         elem->channel_id = entry->channel_id;
 559         elem->event_id = entry->event_id;
 560         /*
 561          * Sanity check :
 562          * We only update the single probe private data when the ptr is
 563          * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
 564          */
 565         WARN_ON(elem->single.func != __mark_empty_function
 566                 && elem->single.probe_private != entry->single.probe_private
 567                 && !elem->ptype);
 568         elem->single.probe_private = entry->single.probe_private;
 569         /*
 570          * Make sure the private data is valid when we update the
 571          * single probe ptr.
 572          */
 573         smp_wmb();
 574         elem->single.func = entry->single.func;
 575         /*
 576          * We also make sure that the new probe callbacks array is consistent
 577          * before setting a pointer to it.
 578          */
 579         rcu_assign_pointer(elem->multi, entry->multi);
 580         /*
 581          * Update the function or multi probe array pointer before setting the
 582          * ptype.
 583          */
 584         smp_wmb();
 585         elem->ptype = entry->ptype;
 586
 587 //ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) {
 588 //ust//         WARN_ON(!elem->tp_cb);
 589 //ust//         /*
 590 //ust//          * It is ok to directly call the probe registration because type
 591 //ust//          * checking has been done in the __trace_mark_tp() macro.
 592 //ust//          */
 593 //ust//
 594 //ust//         if (active) {
 595 //ust//                 /*
 596 //ust//                  * try_module_get should always succeed because we hold
 597 //ust//                  * markers_mutex to get the tp_cb address.
 598 //ust//                  */
 599 //ust//                 ret = try_module_get(__module_text_address(
 600 //ust//                         (unsigned long)elem->tp_cb));
 601 //ust//                 BUG_ON(!ret);
 602 //ust//                 ret = tracepoint_probe_register_noupdate(
 603 //ust//                         elem->tp_name,
 604 //ust//                         elem->tp_cb);
 605 //ust//         } else {
 606 //ust//                 ret = tracepoint_probe_unregister_noupdate(
 607 //ust//                         elem->tp_name,
 608 //ust//                         elem->tp_cb);
 609 //ust//                 /*
 610 //ust//                  * tracepoint_probe_update_all() must be called
 611 //ust//                  * before the module containing tp_cb is unloaded.
 612 //ust//                  */
 613 //ust//                 module_put(__module_text_address(
 614 //ust//                         (unsigned long)elem->tp_cb));
 615 //ust//         }
 616 //ust// }
 617         elem->state__imv = active;
 618
 619         return ret;
 620 }
 621
 622 /*
 623  * Disable a marker and its probe callback.
 624  * Note: only waiting an RCU period after setting elem->call to the empty
 625  * function insures that the original callback is not used anymore. This insured
 626  * by rcu_read_lock_sched around the call site.
 627  */
 628 static void disable_marker(struct marker *elem)
 629 {
 630         int ret;
 631
 632         /* leave "call" as is. It is known statically. */
 633 //ust// if (elem->tp_name && _imv_read(elem->state)) {
 634 //ust//         WARN_ON(!elem->tp_cb);
 635 //ust//         /*
 636 //ust//          * It is ok to directly call the probe registration because type
 637 //ust//          * checking has been done in the __trace_mark_tp() macro.
 638 //ust//          */
 639 //ust//         ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
 640 //ust//                 elem->tp_cb);
 641 //ust//         WARN_ON(ret);
 642 //ust//         /*
 643 //ust//          * tracepoint_probe_update_all() must be called
 644 //ust//          * before the module containing tp_cb is unloaded.
 645 //ust//          */
 646 //ust//         module_put(__module_text_address((unsigned long)elem->tp_cb));
 647 //ust// }
 648         elem->state__imv = 0;
 649         elem->single.func = __mark_empty_function;
 650         /* Update the function before setting the ptype */
 651         smp_wmb();
 652         elem->ptype = 0;        /* single probe */
 653         /*
 654          * Leave the private data and channel_id/event_id there, because removal
 655          * is racy and should be done only after an RCU period. These are never
 656          * used until the next initialization anyway.
 657          */
 658 }
 659
 660 /**
 661  * marker_update_probe_range - Update a probe range
 662  * @begin: beginning of the range
 663  * @end: end of the range
 664  *
 665  * Updates the probe callback corresponding to a range of markers.
 666  */
 667 void marker_update_probe_range(struct marker *begin,
 668         struct marker *end)
 669 {
 670         struct marker *iter;
 671         struct marker_entry *mark_entry;
 672
 673         mutex_lock(&markers_mutex);
 674         for (iter = begin; iter < end; iter++) {
 675                 mark_entry = get_marker(iter->channel, iter->name);
 676                 if (mark_entry) {
 677                         set_marker(mark_entry, iter, !!mark_entry->refcount);
 678                         /*
 679                          * ignore error, continue
 680                          */
 681
 682                         /* This is added for UST. We emit a core_marker_id event
 683                          * for markers that are already registered to a probe
 684                          * upon library load. Otherwise, no core_marker_id will
 685                          * be generated for these markers. Is this the right thing
 686                          * to do?
 687                          */
 688                         trace_mark(metadata, core_marker_id,
 689                                    "channel %s name %s event_id %hu "
 690                                    "int #1u%zu long #1u%zu pointer #1u%zu "
 691                                    "size_t #1u%zu alignment #1u%u",
 692                                    iter->channel, iter->name, mark_entry->event_id,
 693                                    sizeof(int), sizeof(long), sizeof(void *),
 694                                    sizeof(size_t), ltt_get_alignment());
 695                 } else {
 696                         disable_marker(iter);
 697                 }
 698         }
 699         mutex_unlock(&markers_mutex);
 700 }
 701
 702 /*
 703  * Update probes, removing the faulty probes.
 704  *
 705  * Internal callback only changed before the first probe is connected to it.
 706  * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
 707  * transitions.  All other transitions will leave the old private data valid.
 708  * This makes the non-atomicity of the callback/private data updates valid.
 709  *
 710  * "special case" updates :
 711  * 0 -> 1 callback
 712  * 1 -> 0 callback
 713  * 1 -> 2 callbacks
 714  * 2 -> 1 callbacks
 715  * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
 716  * Site effect : marker_set_format may delete the marker entry (creating a
 717  * replacement).
 718  */
 719 static void marker_update_probes(void)
 720 {
 721         /* Core kernel markers */
 722 //ust// marker_update_probe_range(__start___markers, __stop___markers);
 723         /* Markers in modules. */
 724 //ust// module_update_markers();
 725         lib_update_markers();
 726 //ust// tracepoint_probe_update_all();
 727         /* Update immediate values */
 728         core_imv_update();
 729 //ust// module_imv_update();
 730         marker_update_processes();
 731 }
 732
 733 /**
 734  * marker_probe_register -  Connect a probe to a marker
 735  * @channel: marker channel
 736  * @name: marker name
 737  * @format: format string
 738  * @probe: probe handler
 739  * @probe_private: probe private data
 740  *
 741  * private data must be a valid allocated memory address, or NULL.
 742  * Returns 0 if ok, error value on error.
 743  * The probe address must at least be aligned on the architecture pointer size.
 744  */
 745 int marker_probe_register(const char *channel, const char *name,
 746                           const char *format, marker_probe_func *probe,
 747                           void *probe_private)
 748 {
 749         struct marker_entry *entry;
 750         int ret = 0, ret_err;
 751         struct marker_probe_closure *old;
 752         int first_probe = 0;
 753
 754         mutex_lock(&markers_mutex);
 755         entry = get_marker(channel, name);
 756         if (!entry) {
 757                 first_probe = 1;
 758                 entry = add_marker(channel, name, format);
 759                 if (IS_ERR(entry))
 760                         ret = PTR_ERR(entry);
 761                 if (ret)
 762                         goto end;
 763                 ret = ltt_channels_register(channel);
 764                 if (ret)
 765                         goto error_remove_marker;
 766                 ret = ltt_channels_get_index_from_name(channel);
 767                 if (ret < 0)
 768                         goto error_unregister_channel;
 769                 entry->channel_id = ret;
 770                 ret = ltt_channels_get_event_id(channel, name);
 771                 if (ret < 0)
 772                         goto error_unregister_channel;
 773                 entry->event_id = ret;
 774                 ret = 0;
 775                 trace_mark(metadata, core_marker_id,
 776                            "channel %s name %s event_id %hu "
 777                            "int #1u%zu long #1u%zu pointer #1u%zu "
 778                            "size_t #1u%zu alignment #1u%u",
 779                            channel, name, entry->event_id,
 780                            sizeof(int), sizeof(long), sizeof(void *),
 781                            sizeof(size_t), ltt_get_alignment());
 782         } else if (format) {
 783                 if (!entry->format)
 784                         ret = marker_set_format(entry, format);
 785                 else if (strcmp(entry->format, format))
 786                         ret = -EPERM;
 787                 if (ret)
 788                         goto end;
 789         }
 790
 791         /*
 792          * If we detect that a call_rcu is pending for this marker,
 793          * make sure it's executed now.
 794          */
 795         if (entry->rcu_pending)
 796                 rcu_barrier_sched();
 797         old = marker_entry_add_probe(entry, probe, probe_private);
 798         if (IS_ERR(old)) {
 799                 ret = PTR_ERR(old);
 800                 if (first_probe)
 801                         goto error_unregister_channel;
 802                 else
 803                         goto end;
 804         }
 805         mutex_unlock(&markers_mutex);
 806
 807         marker_update_probes();
 808
 809         mutex_lock(&markers_mutex);
 810         entry = get_marker(channel, name);
 811         if (!entry)
 812                 goto end;
 813         if (entry->rcu_pending)
 814                 rcu_barrier_sched();
 815         entry->oldptr = old;
 816         entry->rcu_pending = 1;
 817         /* write rcu_pending before calling the RCU callback */
 818         smp_wmb();
 819         call_rcu_sched(&entry->rcu, free_old_closure);
 820         /*synchronize_rcu(); free_old_closure();*/
 821         goto end;
 822
 823 error_unregister_channel:
 824         ret_err = ltt_channels_unregister(channel);
 825         WARN_ON(ret_err);
 826 error_remove_marker:
 827         ret_err = remove_marker(channel, name);
 828         WARN_ON(ret_err);
 829 end:
 830         mutex_unlock(&markers_mutex);
 831         return ret;
 832 }
 833 //ust// EXPORT_SYMBOL_GPL(marker_probe_register);
 834
 835 /**
 836  * marker_probe_unregister -  Disconnect a probe from a marker
 837  * @channel: marker channel
 838  * @name: marker name
 839  * @probe: probe function pointer
 840  * @probe_private: probe private data
 841  *
 842  * Returns the private data given to marker_probe_register, or an ERR_PTR().
 843  * We do not need to call a synchronize_sched to make sure the probes have
 844  * finished running before doing a module unload, because the module unload
 845  * itself uses stop_machine(), which insures that every preempt disabled section
 846  * have finished.
 847  */
 848 int marker_probe_unregister(const char *channel, const char *name,
 849                             marker_probe_func *probe, void *probe_private)
 850 {
 851         struct marker_entry *entry;
 852         struct marker_probe_closure *old;
 853         int ret = -ENOENT;
 854
 855         mutex_lock(&markers_mutex);
 856         entry = get_marker(channel, name);
 857         if (!entry)
 858                 goto end;
 859         if (entry->rcu_pending)
 860                 rcu_barrier_sched();
 861         old = marker_entry_remove_probe(entry, probe, probe_private);
 862         mutex_unlock(&markers_mutex);
 863
 864         marker_update_probes();
 865
 866         mutex_lock(&markers_mutex);
 867         entry = get_marker(channel, name);
 868         if (!entry)
 869                 goto end;
 870         if (entry->rcu_pending)
 871                 rcu_barrier_sched();
 872         entry->oldptr = old;
 873         entry->rcu_pending = 1;
 874         /* write rcu_pending before calling the RCU callback */
 875         smp_wmb();
 876         call_rcu_sched(&entry->rcu, free_old_closure);
 877         remove_marker(channel, name);   /* Ignore busy error message */
 878         ret = 0;
 879 end:
 880         mutex_unlock(&markers_mutex);
 881         return ret;
 882 }
 883 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister);
 884
 885 static struct marker_entry *
 886 get_marker_from_private_data(marker_probe_func *probe, void *probe_private)
 887 {
 888         struct marker_entry *entry;
 889         unsigned int i;
 890         struct hlist_head *head;
 891         struct hlist_node *node;
 892
 893         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
 894                 head = &marker_table[i];
 895                 hlist_for_each_entry(entry, node, head, hlist) {
 896                         if (!entry->ptype) {
 897                                 if (entry->single.func == probe
 898                                                 && entry->single.probe_private
 899                                                 == probe_private)
 900                                         return entry;
 901                         } else {
 902                                 struct marker_probe_closure *closure;
 903                                 closure = entry->multi;
 904                                 for (i = 0; closure[i].func; i++) {
 905                                         if (closure[i].func == probe &&
 906                                                         closure[i].probe_private
 907                                                         == probe_private)
 908                                                 return entry;
 909                                 }
 910                         }
 911                 }
 912         }
 913         return NULL;
 914 }
 915
 916 /**
 917  * marker_probe_unregister_private_data -  Disconnect a probe from a marker
 918  * @probe: probe function
 919  * @probe_private: probe private data
 920  *
 921  * Unregister a probe by providing the registered private data.
 922  * Only removes the first marker found in hash table.
 923  * Return 0 on success or error value.
 924  * We do not need to call a synchronize_sched to make sure the probes have
 925  * finished running before doing a module unload, because the module unload
 926  * itself uses stop_machine(), which insures that every preempt disabled section
 927  * have finished.
 928  */
 929 int marker_probe_unregister_private_data(marker_probe_func *probe,
 930                 void *probe_private)
 931 {
 932         struct marker_entry *entry;
 933         int ret = 0;
 934         struct marker_probe_closure *old;
 935         const char *channel = NULL, *name = NULL;
 936
 937         mutex_lock(&markers_mutex);
 938         entry = get_marker_from_private_data(probe, probe_private);
 939         if (!entry) {
 940                 ret = -ENOENT;
 941                 goto end;
 942         }
 943         if (entry->rcu_pending)
 944                 rcu_barrier_sched();
 945         old = marker_entry_remove_probe(entry, NULL, probe_private);
 946         channel = kstrdup(entry->channel, GFP_KERNEL);
 947         name = kstrdup(entry->name, GFP_KERNEL);
 948         mutex_unlock(&markers_mutex);
 949
 950         marker_update_probes();
 951
 952         mutex_lock(&markers_mutex);
 953         entry = get_marker(channel, name);
 954         if (!entry)
 955                 goto end;
 956         if (entry->rcu_pending)
 957                 rcu_barrier_sched();
 958         entry->oldptr = old;
 959         entry->rcu_pending = 1;
 960         /* write rcu_pending before calling the RCU callback */
 961         smp_wmb();
 962         call_rcu_sched(&entry->rcu, free_old_closure);
 963         /* Ignore busy error message */
 964         remove_marker(channel, name);
 965 end:
 966         mutex_unlock(&markers_mutex);
 967         kfree(channel);
 968         kfree(name);
 969         return ret;
 970 }
 971 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
 972
 973 /**
 974  * marker_get_private_data - Get a marker's probe private data
 975  * @channel: marker channel
 976  * @name: marker name
 977  * @probe: probe to match
 978  * @num: get the nth matching probe's private data
 979  *
 980  * Returns the nth private data pointer (starting from 0) matching, or an
 981  * ERR_PTR.
 982  * Returns the private data pointer, or an ERR_PTR.
 983  * The private data pointer should _only_ be dereferenced if the caller is the
 984  * owner of the data, or its content could vanish. This is mostly used to
 985  * confirm that a caller is the owner of a registered probe.
 986  */
 987 void *marker_get_private_data(const char *channel, const char *name,
 988                               marker_probe_func *probe, int num)
 989 {
 990         struct hlist_head *head;
 991         struct hlist_node *node;
 992         struct marker_entry *e;
 993         size_t channel_len = strlen(channel) + 1;
 994         size_t name_len = strlen(name) + 1;
 995         int i;
 996         u32 hash;
 997
 998         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 999         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
1000         hlist_for_each_entry(e, node, head, hlist) {
1001                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
1002                         if (!e->ptype) {
1003                                 if (num == 0 && e->single.func == probe)
1004                                         return e->single.probe_private;
1005                         } else {
1006                                 struct marker_probe_closure *closure;
1007                                 int match = 0;
1008                                 closure = e->multi;
1009                                 for (i = 0; closure[i].func; i++) {
1010                                         if (closure[i].func != probe)
1011                                                 continue;
1012                                         if (match++ == num)
1013                                                 return closure[i].probe_private;
1014                                 }
1015                         }
1016                         break;
1017                 }
1018         }
1019         return ERR_PTR(-ENOENT);
1020 }
1021 //ust// EXPORT_SYMBOL_GPL(marker_get_private_data);
1022
1023 /**
1024  * markers_compact_event_ids - Compact markers event IDs and reassign channels
1025  *
1026  * Called when no channel users are active by the channel infrastructure.
1027  * Called with lock_markers() and channel mutex held.
1028  */
1029 //ust// void markers_compact_event_ids(void)
1030 //ust// {
1031 //ust//         struct marker_entry *entry;
1032 //ust//         unsigned int i;
1033 //ust//         struct hlist_head *head;
1034 //ust//         struct hlist_node *node;
1035 //ust//         int ret;
1036 //ust//
1037 //ust//         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
1038 //ust//                 head = &marker_table[i];
1039 //ust//                 hlist_for_each_entry(entry, node, head, hlist) {
1040 //ust//                         ret = ltt_channels_get_index_from_name(entry->channel);
1041 //ust//                         WARN_ON(ret < 0);
1042 //ust//                         entry->channel_id = ret;
1043 //ust//                         ret = _ltt_channels_get_event_id(entry->channel,
1044 //ust//                                                          entry->name);
1045 //ust//                         WARN_ON(ret < 0);
1046 //ust//                         entry->event_id = ret;
1047 //ust//                 }
1048 //ust//         }
1049 //ust// }
1050
1051 //ust//#ifdef CONFIG_MODULES
1052
1053 /**
1054  * marker_get_iter_range - Get a next marker iterator given a range.
1055  * @marker: current markers (in), next marker (out)
1056  * @begin: beginning of the range
1057  * @end: end of the range
1058  *
1059  * Returns whether a next marker has been found (1) or not (0).
1060  * Will return the first marker in the range if the input marker is NULL.
1061  */
1062 int marker_get_iter_range(struct marker **marker, struct marker *begin,
1063         struct marker *end)
1064 {
1065         if (!*marker && begin != end) {
1066                 *marker = begin;
1067                 return 1;
1068         }
1069         if (*marker >= begin && *marker < end)
1070                 return 1;
1071         return 0;
1072 }
1073 //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range);
1074
1075 static void marker_get_iter(struct marker_iter *iter)
1076 {
1077         int found = 0;
1078
1079         /* Core kernel markers */
1080         if (!iter->lib) {
1081                 /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */
1082                 found = marker_get_iter_range(&iter->marker,
1083                                 __start___markers, __stop___markers);
1084                 if (found)
1085                         goto end;
1086         }
1087         /* Markers in modules. */
1088         found = lib_get_iter_markers(iter);
1089 end:
1090         if (!found)
1091                 marker_iter_reset(iter);
1092 }
1093
1094 void marker_iter_start(struct marker_iter *iter)
1095 {
1096         marker_get_iter(iter);
1097 }
1098 //ust// EXPORT_SYMBOL_GPL(marker_iter_start);
1099
1100 void marker_iter_next(struct marker_iter *iter)
1101 {
1102         iter->marker++;
1103         /*
1104          * iter->marker may be invalid because we blindly incremented it.
1105          * Make sure it is valid by marshalling on the markers, getting the
1106          * markers from following modules if necessary.
1107          */
1108         marker_get_iter(iter);
1109 }
1110 //ust// EXPORT_SYMBOL_GPL(marker_iter_next);
1111
1112 void marker_iter_stop(struct marker_iter *iter)
1113 {
1114 }
1115 //ust// EXPORT_SYMBOL_GPL(marker_iter_stop);
1116
1117 void marker_iter_reset(struct marker_iter *iter)
1118 {
1119         iter->lib = NULL;
1120         iter->marker = NULL;
1121 }
1122 //ust// EXPORT_SYMBOL_GPL(marker_iter_reset);
1123
1124 #ifdef CONFIG_MARKERS_USERSPACE
1125 /*
1126  * must be called with current->user_markers_mutex held
1127  */
1128 static void free_user_marker(char __user *state, struct hlist_head *head)
1129 {
1130         struct user_marker *umark;
1131         struct hlist_node *pos, *n;
1132
1133         hlist_for_each_entry_safe(umark, pos, n, head, hlist) {
1134                 if (umark->state == state) {
1135                         hlist_del(&umark->hlist);
1136                         kfree(umark);
1137                 }
1138         }
1139 }
1140
1141 //ust// asmlinkage long sys_marker(char __user *name, char __user *format,
1142 //ust//                 char __user *state, int reg)
1143 //ust// {
1144 //ust//         struct user_marker *umark;
1145 //ust//         long len;
1146 //ust//         struct marker_entry *entry;
1147 //ust//         int ret = 0;
1148 //ust//
1149 //ust//         printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n",
1150 //ust//                 current->comm, reg ? "registers" : "unregisters",
1151 //ust//                 name, state);
1152 //ust//         if (reg) {
1153 //ust//                 umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL);
1154 //ust//                 umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0';
1155 //ust//                 umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0';
1156 //ust//                 umark->state = state;
1157 //ust//                 len = strncpy_from_user(umark->name, name,
1158 //ust//                         MAX_USER_MARKER_NAME_LEN - 1);
1159 //ust//                 if (len < 0) {
1160 //ust//                         ret = -EFAULT;
1161 //ust//                         goto error;
1162 //ust//                 }
1163 //ust//                 len = strncpy_from_user(umark->format, format,
1164 //ust//                         MAX_USER_MARKER_FORMAT_LEN - 1);
1165 //ust//                 if (len < 0) {
1166 //ust//                         ret = -EFAULT;
1167 //ust//                         goto error;
1168 //ust//                 }
1169 //ust//                 printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name,
1170 //ust//                         umark->format);
1171 //ust//                 mutex_lock(&markers_mutex);
1172 //ust//                 entry = get_marker("userspace", umark->name);
1173 //ust//                 if (entry) {
1174 //ust//                         if (entry->format &&
1175 //ust//                                 strcmp(entry->format, umark->format) != 0) {
1176 //ust//                                 printk(" error, wrong format in process %s",
1177 //ust//                                         current->comm);
1178 //ust//                                 ret = -EPERM;
1179 //ust//                                 goto error_unlock;
1180 //ust//                         }
1181 //ust//                         printk(" %s", !!entry->refcount
1182 //ust//                                         ? "enabled" : "disabled");
1183 //ust//                         if (put_user(!!entry->refcount, state)) {
1184 //ust//                                 ret = -EFAULT;
1185 //ust//                                 goto error_unlock;
1186 //ust//                         }
1187 //ust//                         printk("\n");
1188 //ust//                 } else {
1189 //ust//                         printk(" disabled\n");
1190 //ust//                         if (put_user(0, umark->state)) {
1191 //ust//                                 printk(KERN_WARNING
1192 //ust//                                         "Marker in %s caused a fault\n",
1193 //ust//                                         current->comm);
1194 //ust//                                 goto error_unlock;
1195 //ust//                         }
1196 //ust//                 }
1197 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1198 //ust//                 hlist_add_head(&umark->hlist,
1199 //ust//                         &current->group_leader->user_markers);
1200 //ust//                 current->group_leader->user_markers_sequence++;
1201 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1202 //ust//                 mutex_unlock(&markers_mutex);
1203 //ust//         } else {
1204 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1205 //ust//                 free_user_marker(state,
1206 //ust//                         &current->group_leader->user_markers);
1207 //ust//                 current->group_leader->user_markers_sequence++;
1208 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1209 //ust//         }
1210 //ust//         goto end;
1211 //ust// error_unlock:
1212 //ust//         mutex_unlock(&markers_mutex);
1213 //ust// error:
1214 //ust//         kfree(umark);
1215 //ust// end:
1216 //ust//         return ret;
1217 //ust// }
1218 //ust//
1219 //ust// /*
1220 //ust//  * Types :
1221 //ust//  * string : 0
1222 //ust//  */
1223 //ust// asmlinkage long sys_trace(int type, uint16_t id,
1224 //ust//                 char __user *ubuf)
1225 //ust// {
1226 //ust//         long ret = -EPERM;
1227 //ust//         char *page;
1228 //ust//         int len;
1229 //ust//
1230 //ust//         switch (type) {
1231 //ust//         case 0: /* String */
1232 //ust//                 ret = -ENOMEM;
1233 //ust//                 page = (char *)__get_free_page(GFP_TEMPORARY);
1234 //ust//                 if (!page)
1235 //ust//                         goto string_out;
1236 //ust//                 len = strncpy_from_user(page, ubuf, PAGE_SIZE);
1237 //ust//                 if (len < 0) {
1238 //ust//                         ret = -EFAULT;
1239 //ust//                         goto string_err;
1240 //ust//                 }
1241 //ust//                 trace_mark(userspace, string, "string %s", page);
1242 //ust// string_err:
1243 //ust//                 free_page((unsigned long) page);
1244 //ust// string_out:
1245 //ust//                 break;
1246 //ust//         default:
1247 //ust//                 break;
1248 //ust//         }
1249 //ust//         return ret;
1250 //ust// }
1251
1252 //ust// static void marker_update_processes(void)
1253 //ust// {
1254 //ust//         struct task_struct *g, *t;
1255 //ust//
1256 //ust//         /*
1257 //ust//          * markers_mutex is taken to protect the p->user_markers read.
1258 //ust//          */
1259 //ust//         mutex_lock(&markers_mutex);
1260 //ust//         read_lock(&tasklist_lock);
1261 //ust//         for_each_process(g) {
1262 //ust//                 WARN_ON(!thread_group_leader(g));
1263 //ust//                 if (hlist_empty(&g->user_markers))
1264 //ust//                         continue;
1265 //ust//                 if (strcmp(g->comm, "testprog") == 0)
1266 //ust//                         printk(KERN_DEBUG "set update pending for testprog\n");
1267 //ust//                 t = g;
1268 //ust//                 do {
1269 //ust//                         /* TODO : implement this thread flag in each arch. */
1270 //ust//                         set_tsk_thread_flag(t, TIF_MARKER_PENDING);
1271 //ust//                 } while ((t = next_thread(t)) != g);
1272 //ust//         }
1273 //ust//         read_unlock(&tasklist_lock);
1274 //ust//         mutex_unlock(&markers_mutex);
1275 //ust// }
1276
1277 /*
1278  * Update current process.
1279  * Note that we have to wait a whole scheduler period before we are sure that
1280  * every running userspace threads have their markers updated.
1281  * (synchronize_sched() can be used to insure this).
1282  */
1283 void marker_update_process(void)
1284 {
1285         struct user_marker *umark;
1286         struct hlist_node *pos;
1287         struct marker_entry *entry;
1288
1289         mutex_lock(&markers_mutex);
1290         mutex_lock(&current->group_leader->user_markers_mutex);
1291         if (strcmp(current->comm, "testprog") == 0)
1292                 printk(KERN_DEBUG "do update pending for testprog\n");
1293         hlist_for_each_entry(umark, pos,
1294                         &current->group_leader->user_markers, hlist) {
1295                 printk(KERN_DEBUG "Updating marker %s in %s\n",
1296                         umark->name, current->comm);
1297                 entry = get_marker("userspace", umark->name);
1298                 if (entry) {
1299                         if (entry->format &&
1300                                 strcmp(entry->format, umark->format) != 0) {
1301                                 printk(KERN_WARNING
1302                                         " error, wrong format in process %s\n",
1303                                         current->comm);
1304                                 break;
1305                         }
1306                         if (put_user(!!entry->refcount, umark->state)) {
1307                                 printk(KERN_WARNING
1308                                         "Marker in %s caused a fault\n",
1309                                         current->comm);
1310                                 break;
1311                         }
1312                 } else {
1313                         if (put_user(0, umark->state)) {
1314                                 printk(KERN_WARNING
1315                                         "Marker in %s caused a fault\n",
1316                                         current->comm);
1317                                 break;
1318                         }
1319                 }
1320         }
1321         clear_thread_flag(TIF_MARKER_PENDING);
1322         mutex_unlock(&current->group_leader->user_markers_mutex);
1323         mutex_unlock(&markers_mutex);
1324 }
1325
1326 /*
1327  * Called at process exit and upon do_execve().
1328  * We assume that when the leader exits, no more references can be done to the
1329  * leader structure by the other threads.
1330  */
1331 void exit_user_markers(struct task_struct *p)
1332 {
1333         struct user_marker *umark;
1334         struct hlist_node *pos, *n;
1335
1336         if (thread_group_leader(p)) {
1337                 mutex_lock(&markers_mutex);
1338                 mutex_lock(&p->user_markers_mutex);
1339                 hlist_for_each_entry_safe(umark, pos, n, &p->user_markers,
1340                         hlist)
1341                     kfree(umark);
1342                 INIT_HLIST_HEAD(&p->user_markers);
1343                 p->user_markers_sequence++;
1344                 mutex_unlock(&p->user_markers_mutex);
1345                 mutex_unlock(&markers_mutex);
1346         }
1347 }
1348
1349 int is_marker_enabled(const char *channel, const char *name)
1350 {
1351         struct marker_entry *entry;
1352
1353         mutex_lock(&markers_mutex);
1354         entry = get_marker(channel, name);
1355         mutex_unlock(&markers_mutex);
1356
1357         return entry && !!entry->refcount;
1358 }
1359 //ust// #endif
1360
1361 int marker_module_notify(struct notifier_block *self,
1362                          unsigned long val, void *data)
1363 {
1364         struct module *mod = data;
1365
1366         switch (val) {
1367         case MODULE_STATE_COMING:
1368                 marker_update_probe_range(mod->markers,
1369                         mod->markers + mod->num_markers);
1370                 break;
1371         case MODULE_STATE_GOING:
1372                 marker_update_probe_range(mod->markers,
1373                         mod->markers + mod->num_markers);
1374                 break;
1375         }
1376         return 0;
1377 }
1378
1379 struct notifier_block marker_module_nb = {
1380         .notifier_call = marker_module_notify,
1381         .priority = 0,
1382 };
1383
1384 //ust// static int init_markers(void)
1385 //ust// {
1386 //ust//         return register_module_notifier(&marker_module_nb);
1387 //ust// }
1388 //ust// __initcall(init_markers);
1389 /* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */
1390
1391 #endif /* CONFIG_MODULES */
1392
1393 void ltt_dump_marker_state(struct ltt_trace_struct *trace)
1394 {
1395         struct marker_iter iter;
1396         struct ltt_probe_private_data call_data;
1397         const char *channel;
1398
1399         call_data.trace = trace;
1400         call_data.serializer = NULL;
1401
1402         marker_iter_reset(&iter);
1403         marker_iter_start(&iter);
1404         for (; iter.marker != NULL; marker_iter_next(&iter)) {
1405                 if (!_imv_read(iter.marker->state))
1406                         continue;
1407                 channel = ltt_channels_get_name_from_index(
1408                                 iter.marker->channel_id);
1409                 __trace_mark(0, metadata, core_marker_id,
1410                         &call_data,
1411                         "channel %s name %s event_id %hu "
1412                         "int #1u%zu long #1u%zu pointer #1u%zu "
1413                         "size_t #1u%zu alignment #1u%u",
1414                         channel,
1415                         iter.marker->name,
1416                         iter.marker->event_id,
1417                         sizeof(int), sizeof(long),
1418                         sizeof(void *), sizeof(size_t),
1419                         ltt_get_alignment());
1420                 if (iter.marker->format)
1421                         __trace_mark(0, metadata,
1422                                 core_marker_format,
1423                                 &call_data,
1424                                 "channel %s name %s format %s",
1425                                 channel,
1426                                 iter.marker->name,
1427                                 iter.marker->format);
1428         }
1429         marker_iter_stop(&iter);
1430 }
1431 //ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state);
1432
1433
1434 static LIST_HEAD(libs);
1435
1436 /*
1437  * Returns 0 if current not found.
1438  * Returns 1 if current found.
1439  */
1440 int lib_get_iter_markers(struct marker_iter *iter)
1441 {
1442         struct lib *iter_lib;
1443         int found = 0;
1444
1445 //ust// mutex_lock(&module_mutex);
1446         list_for_each_entry(iter_lib, &libs, list) {
1447                 if (iter_lib < iter->lib)
1448                         continue;
1449                 else if (iter_lib > iter->lib)
1450                         iter->marker = NULL;
1451                 found = marker_get_iter_range(&iter->marker,
1452                         iter_lib->markers_start,
1453                         iter_lib->markers_start + iter_lib->markers_count);
1454                 if (found) {
1455                         iter->lib = iter_lib;
1456                         break;
1457                 }
1458         }
1459 //ust// mutex_unlock(&module_mutex);
1460         return found;
1461 }
1462
1463 void lib_update_markers(void)
1464 {
1465         struct lib *lib;
1466
1467 //ust// mutex_lock(&module_mutex);
1468         list_for_each_entry(lib, &libs, list)
1469                 marker_update_probe_range(lib->markers_start,
1470                                 lib->markers_start + lib->markers_count);
1471 //ust// mutex_unlock(&module_mutex);
1472 }
1473
1474 static void (*new_marker_cb)(struct marker *) = NULL;
1475
1476 void marker_set_new_marker_cb(void (*cb)(struct marker *))
1477 {
1478         new_marker_cb = cb;
1479 }
1480
1481 static void new_markers(struct marker *start, struct marker *end)
1482 {
1483         if(new_marker_cb) {
1484                 struct marker *m;
1485                 for(m=start; m < end; m++) {
1486                         new_marker_cb(m);
1487                 }
1488         }
1489 }
1490
1491 int marker_register_lib(struct marker *markers_start, int markers_count)
1492 {
1493         struct lib *pl;
1494
1495         pl = (struct lib *) malloc(sizeof(struct lib));
1496
1497         pl->markers_start = markers_start;
1498         pl->markers_count = markers_count;
1499
1500         list_add(&pl->list, &libs);
1501
1502         new_markers(markers_start, markers_start + markers_count);
1503
1504         /* FIXME: update just the loaded lib */
1505         lib_update_markers();
1506
1507         DBG("just registered a markers section from %p and having %d markers", markers_start, markers_count);
1508
1509         return 0;
1510 }
1511
1512 static int initialized = 0;
1513
1514 void __attribute__((constructor)) init_markers(void)
1515 {
1516         if(!initialized) {
1517                 marker_register_lib(__start___markers, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker));
1518                 printf("markers_start: %p, markers_stop: %p\n", __start___markers, __stop___markers);
1519                 initialized = 1;
1520         }
1521 }