2 * Copyright (C) 2007-2011 Mathieu Desnoyers
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation;
7 * version 2.1 of the License.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <urcu/rculist.h>
24 #include <urcu/hlist.h>
27 #include <ust/marker.h>
28 #include <ust/marker-internal.h>
29 #include <ust/tracepoint.h>
30 #include <ust/tracepoint-internal.h>
32 #include "usterr_signal_safe.h"
34 #include "tracercore.h"
37 __thread
long ust_reg_stack
[500];
38 volatile __thread
long *ust_reg_stack_ptr
= (long *) 0;
40 extern struct ust_marker
* const __start___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
41 extern struct ust_marker
* const __stop___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
43 /* Set to 1 to enable ust_marker debug output */
44 static const int ust_marker_debug
;
47 * ust_marker_mutex nests inside module_mutex. ust_marker mutex protects
48 * the builtin and module ust_marker and the hash table.
50 static DEFINE_MUTEX(ust_marker_mutex
);
52 static CDS_LIST_HEAD(ust_marker_libs
);
55 void lock_ust_marker(void)
57 pthread_mutex_lock(&ust_marker_mutex
);
60 void unlock_ust_marker(void)
62 pthread_mutex_unlock(&ust_marker_mutex
);
66 * ust_marker hash table, containing the active ust_marker.
67 * Protected by module_mutex.
69 #define UST_MARKER_HASH_BITS 6
70 #define UST_MARKER_TABLE_SIZE (1 << UST_MARKER_HASH_BITS)
71 static struct cds_hlist_head ust_marker_table
[UST_MARKER_TABLE_SIZE
];
75 * It is used to make sure every handler has finished using its private
76 * data between two consecutive operation (add or remove) on a given
77 * ust_marker. It is also used to delay the free of multiple probes
78 * array until a quiescent state is reached. ust_marker entries
79 * modifications are protected by the ust_marker_mutex.
81 struct ust_marker_entry
{
82 struct cds_hlist_node hlist
;
86 void (*call
)(const struct ust_marker
*mdata
, void *call_private
, ...);
87 struct ust_marker_probe_closure single
;
88 struct ust_marker_probe_closure
*multi
;
89 int refcount
; /* Number of times armed. 0 if disarmed. */
95 unsigned char ptype
:1;
96 unsigned char format_allocated
:1;
97 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
100 #ifdef CONFIG_UST_MARKER_USERSPACE
101 static void ust_marker_update_processes(void);
103 static void ust_marker_update_processes(void)
109 * __ust_marker_empty_function - Empty probe callback
110 * @mdata: ust_marker data
111 * @probe_private: probe private data
112 * @call_private: call site private data
113 * @fmt: format string
114 * @...: variable argument list
116 * Empty callback provided as a probe to the ust_marker. By providing
117 * this to a disabled ust_marker, we make sure the execution flow is
118 * always valid even though the function pointer change and the
119 * ust_marker enabling are two distinct operations that modifies the
120 * execution flow of preemptible code.
122 notrace
void __ust_marker_empty_function(const struct ust_marker
*mdata
,
123 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
126 //ust// EXPORT_SYMBOL_GPL(__ust_marker_empty_function);
129 * ust_marker_probe_cb Callback that prepares the variable argument list for probes.
130 * @mdata: pointer of type struct ust_marker
131 * @call_private: caller site private data
132 * @...: Variable argument list.
134 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
135 * need to put a full cmm_smp_rmb() in this branch. This is why we do not use
136 * rcu_dereference() for the pointer read.
138 notrace
void ust_marker_probe_cb(const struct ust_marker
*mdata
,
139 void *call_private
, ...)
145 * rcu_read_lock_sched does two things : disabling preemption to make
146 * sure the teardown of the callbacks can be done correctly when they
147 * are in modules and they insure RCU read coherency.
149 //ust// rcu_read_lock_sched_notrace();
150 ptype
= mdata
->ptype
;
151 if (likely(!ptype
)) {
152 ust_marker_probe_func
*func
;
153 /* Must read the ptype before ptr. They are not data dependant,
154 * so we put an explicit cmm_smp_rmb() here. */
156 func
= mdata
->single
.func
;
157 /* Must read the ptr before private data. They are not data
158 * dependant, so we put an explicit cmm_smp_rmb() here. */
160 va_start(args
, call_private
);
161 func(mdata
, mdata
->single
.probe_private
, call_private
,
162 mdata
->format
, &args
);
165 struct ust_marker_probe_closure
*multi
;
168 * Read mdata->ptype before mdata->multi.
171 multi
= mdata
->multi
;
173 * multi points to an array, therefore accessing the array
174 * depends on reading multi. However, even in this case,
175 * we must insure that the pointer is read _before_ the array
176 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
177 * in the fast path, so put the explicit cmm_barrier here.
179 cmm_smp_read_barrier_depends();
180 for (i
= 0; multi
[i
].func
; i
++) {
181 va_start(args
, call_private
);
182 multi
[i
].func(mdata
, multi
[i
].probe_private
,
183 call_private
, mdata
->format
, &args
);
187 //ust// rcu_read_unlock_sched_notrace();
189 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_cb);
192 * ust_marker_probe_cb Callback that does not prepare the variable argument list.
193 * @mdata: pointer of type struct ust_marker
194 * @call_private: caller site private data
195 * @...: Variable argument list.
197 * Should be connected to ust_marker "UST_MARKER_NOARGS".
199 static notrace
void ust_marker_probe_cb_noarg(const struct ust_marker
*mdata
,
200 void *call_private
, ...)
202 va_list args
; /* not initialized */
205 //ust// rcu_read_lock_sched_notrace();
206 ptype
= mdata
->ptype
;
207 if (likely(!ptype
)) {
208 ust_marker_probe_func
*func
;
209 /* Must read the ptype before ptr. They are not data dependant,
210 * so we put an explicit cmm_smp_rmb() here. */
212 func
= mdata
->single
.func
;
213 /* Must read the ptr before private data. They are not data
214 * dependant, so we put an explicit cmm_smp_rmb() here. */
216 func(mdata
, mdata
->single
.probe_private
, call_private
,
217 mdata
->format
, &args
);
219 struct ust_marker_probe_closure
*multi
;
222 * Read mdata->ptype before mdata->multi.
225 multi
= mdata
->multi
;
227 * multi points to an array, therefore accessing the array
228 * depends on reading multi. However, even in this case,
229 * we must insure that the pointer is read _before_ the array
230 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
231 * in the fast path, so put the explicit cmm_barrier here.
233 cmm_smp_read_barrier_depends();
234 for (i
= 0; multi
[i
].func
; i
++)
235 multi
[i
].func(mdata
, multi
[i
].probe_private
,
236 call_private
, mdata
->format
, &args
);
238 //ust// rcu_read_unlock_sched_notrace();
241 static void free_old_closure(struct rcu_head
*head
)
243 struct ust_marker_entry
*entry
= _ust_container_of(head
,
244 struct ust_marker_entry
, rcu
);
246 /* Make sure we free the data before setting the pending flag to 0 */
248 entry
->rcu_pending
= 0;
251 static void debug_print_probes(struct ust_marker_entry
*entry
)
255 if (!ust_marker_debug
)
259 DBG("Single probe : %p %p",
261 entry
->single
.probe_private
);
263 for (i
= 0; entry
->multi
[i
].func
; i
++)
264 DBG("Multi probe %d : %p %p", i
,
265 entry
->multi
[i
].func
,
266 entry
->multi
[i
].probe_private
);
270 static struct ust_marker_probe_closure
*
271 ust_marker_entry_add_probe(struct ust_marker_entry
*entry
,
272 ust_marker_probe_func
*probe
, void *probe_private
)
275 struct ust_marker_probe_closure
*old
, *new;
279 debug_print_probes(entry
);
282 if (entry
->single
.func
== probe
&&
283 entry
->single
.probe_private
== probe_private
)
284 return ERR_PTR(-EBUSY
);
285 if (entry
->single
.func
== __ust_marker_empty_function
) {
287 entry
->single
.func
= probe
;
288 entry
->single
.probe_private
= probe_private
;
291 debug_print_probes(entry
);
299 /* (N -> N+1), (N != 0, 1) probes */
300 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++)
301 if (old
[nr_probes
].func
== probe
302 && old
[nr_probes
].probe_private
304 return ERR_PTR(-EBUSY
);
306 /* + 2 : one for new probe, one for NULL func */
307 new = zmalloc((nr_probes
+ 2) * sizeof(struct ust_marker_probe_closure
));
309 return ERR_PTR(-ENOMEM
);
311 new[0] = entry
->single
;
314 nr_probes
* sizeof(struct ust_marker_probe_closure
));
315 new[nr_probes
].func
= probe
;
316 new[nr_probes
].probe_private
= probe_private
;
317 entry
->refcount
= nr_probes
+ 1;
320 debug_print_probes(entry
);
324 static struct ust_marker_probe_closure
*
325 ust_marker_entry_remove_probe(struct ust_marker_entry
*entry
,
326 ust_marker_probe_func
*probe
, void *probe_private
)
328 int nr_probes
= 0, nr_del
= 0, i
;
329 struct ust_marker_probe_closure
*old
, *new;
333 debug_print_probes(entry
);
335 /* 0 -> N is an error */
336 WARN_ON(entry
->single
.func
== __ust_marker_empty_function
);
338 WARN_ON(probe
&& entry
->single
.func
!= probe
);
339 WARN_ON(entry
->single
.probe_private
!= probe_private
);
340 entry
->single
.func
= __ust_marker_empty_function
;
343 debug_print_probes(entry
);
346 /* (N -> M), (N > 1, M >= 0) probes */
347 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
348 if ((!probe
|| old
[nr_probes
].func
== probe
)
349 && old
[nr_probes
].probe_private
355 if (nr_probes
- nr_del
== 0) {
356 /* N -> 0, (N > 1) */
357 entry
->single
.func
= __ust_marker_empty_function
;
360 } else if (nr_probes
- nr_del
== 1) {
361 /* N -> 1, (N > 1) */
362 for (i
= 0; old
[i
].func
; i
++)
363 if ((probe
&& old
[i
].func
!= probe
) ||
364 old
[i
].probe_private
!= probe_private
)
365 entry
->single
= old
[i
];
370 /* N -> M, (N > 1, M > 1) */
372 new = zmalloc((nr_probes
- nr_del
+ 1) * sizeof(struct ust_marker_probe_closure
));
374 return ERR_PTR(-ENOMEM
);
375 for (i
= 0; old
[i
].func
; i
++)
376 if ((probe
&& old
[i
].func
!= probe
) ||
377 old
[i
].probe_private
!= probe_private
)
379 entry
->refcount
= nr_probes
- nr_del
;
383 debug_print_probes(entry
);
388 * Get ust_marker if the ust_marker is present in the ust_marker hash table.
389 * Must be called with ust_marker_mutex held.
390 * Returns NULL if not present.
392 static struct ust_marker_entry
*get_ust_marker(const char *channel
, const char *name
)
394 struct cds_hlist_head
*head
;
395 struct cds_hlist_node
*node
;
396 struct ust_marker_entry
*e
;
397 size_t channel_len
= strlen(channel
) + 1;
398 size_t name_len
= strlen(name
) + 1;
401 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
402 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
403 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
404 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
411 * Add the ust_marker to the ust_marker hash table. Must be called with
412 * ust_marker_mutex held.
414 static struct ust_marker_entry
*add_ust_marker(const char *channel
, const char *name
,
417 struct cds_hlist_head
*head
;
418 struct cds_hlist_node
*node
;
419 struct ust_marker_entry
*e
;
420 size_t channel_len
= strlen(channel
) + 1;
421 size_t name_len
= strlen(name
) + 1;
422 size_t format_len
= 0;
425 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
427 format_len
= strlen(format
) + 1;
428 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
429 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
430 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
431 DBG("ust_marker %s.%s busy", channel
, name
);
432 return ERR_PTR(-EBUSY
); /* Already there */
436 * Using zmalloc here to allocate a variable length element. Could
437 * cause some memory fragmentation if overused.
439 e
= zmalloc(sizeof(struct ust_marker_entry
)
440 + channel_len
+ name_len
+ format_len
);
442 return ERR_PTR(-ENOMEM
);
443 memcpy(e
->channel
, channel
, channel_len
);
444 e
->name
= &e
->channel
[channel_len
];
445 memcpy(e
->name
, name
, name_len
);
447 e
->format
= &e
->name
[name_len
];
448 memcpy(e
->format
, format
, format_len
);
449 if (strcmp(e
->format
, UST_MARKER_NOARGS
) == 0)
450 e
->call
= ust_marker_probe_cb_noarg
;
452 e
->call
= ust_marker_probe_cb
;
453 __ust_marker(metadata
, core_marker_format
, NULL
,
454 "channel %s name %s format %s",
455 e
->channel
, e
->name
, e
->format
);
458 e
->call
= ust_marker_probe_cb
;
460 e
->single
.func
= __ust_marker_empty_function
;
461 e
->single
.probe_private
= NULL
;
464 e
->format_allocated
= 0;
467 cds_hlist_add_head(&e
->hlist
, head
);
472 * Remove the ust_marker from the ust_marker hash table. Must be called with mutex_lock
475 static int remove_ust_marker(const char *channel
, const char *name
)
477 struct cds_hlist_head
*head
;
478 struct cds_hlist_node
*node
;
479 struct ust_marker_entry
*e
;
481 size_t channel_len
= strlen(channel
) + 1;
482 size_t name_len
= strlen(name
) + 1;
486 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
487 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
488 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
489 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
496 if (e
->single
.func
!= __ust_marker_empty_function
)
498 cds_hlist_del(&e
->hlist
);
499 if (e
->format_allocated
)
501 ret
= ltt_channels_unregister(e
->channel
);
503 /* Make sure the call_rcu has been executed */
504 //ust// if (e->rcu_pending)
505 //ust// rcu_cmm_barrier_sched();
511 * Set the mark_entry format to the format found in the element.
513 static int ust_marker_set_format(struct ust_marker_entry
*entry
, const char *format
)
515 entry
->format
= strdup(format
);
518 entry
->format_allocated
= 1;
520 __ust_marker(metadata
, core_marker_format
, NULL
,
521 "channel %s name %s format %s",
522 entry
->channel
, entry
->name
, entry
->format
);
527 * Sets the probe callback corresponding to one ust_marker.
529 static int set_ust_marker(struct ust_marker_entry
*entry
, struct ust_marker
*elem
,
533 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
536 if (strcmp(entry
->format
, elem
->format
) != 0) {
537 ERR("Format mismatch for probe %s (%s), ust_marker (%s)",
544 ret
= ust_marker_set_format(entry
, elem
->format
);
550 * probe_cb setup (statically known) is done here. It is
551 * asynchronous with the rest of execution, therefore we only
552 * pass from a "safe" callback (with argument) to an "unsafe"
553 * callback (does not set arguments).
555 elem
->call
= entry
->call
;
556 elem
->channel_id
= entry
->channel_id
;
557 elem
->event_id
= entry
->event_id
;
560 * We only update the single probe private data when the ptr is
561 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
563 WARN_ON(elem
->single
.func
!= __ust_marker_empty_function
564 && elem
->single
.probe_private
!= entry
->single
.probe_private
566 elem
->single
.probe_private
= entry
->single
.probe_private
;
568 * Make sure the private data is valid when we update the
572 elem
->single
.func
= entry
->single
.func
;
574 * We also make sure that the new probe callbacks array is consistent
575 * before setting a pointer to it.
577 rcu_assign_pointer(elem
->multi
, entry
->multi
);
579 * Update the function or multi probe array pointer before setting the
583 elem
->ptype
= entry
->ptype
;
585 if (elem
->tp_name
&& (active
^ elem
->state
)) {
586 WARN_ON(!elem
->tp_cb
);
588 * It is ok to directly call the probe registration because type
589 * checking has been done in the __ust_marker_tp() macro.
594 * try_module_get should always succeed because we hold
595 * ust_marker_mutex to get the tp_cb address.
597 //ust// ret = try_module_get(__module_text_address(
598 //ust// (unsigned long)elem->tp_cb));
599 //ust// BUG_ON(!ret);
600 ret
= tracepoint_probe_register_noupdate(
604 ret
= tracepoint_probe_unregister_noupdate(
608 * tracepoint_probe_update_all() must be called
609 * before the module containing tp_cb is unloaded.
611 //ust// module_put(__module_text_address(
612 //ust// (unsigned long)elem->tp_cb));
615 elem
->state
= active
;
621 * Disable a ust_marker and its probe callback.
622 * Note: only waiting an RCU period after setting elem->call to the empty
623 * function insures that the original callback is not used anymore. This insured
624 * by rcu_read_lock_sched around the call site.
626 static void disable_ust_marker(struct ust_marker
*elem
)
630 /* leave "call" as is. It is known statically. */
631 if (elem
->tp_name
&& elem
->state
) {
632 WARN_ON(!elem
->tp_cb
);
634 * It is ok to directly call the probe registration because type
635 * checking has been done in the __ust_marker_tp() macro.
637 ret
= tracepoint_probe_unregister_noupdate(elem
->tp_name
,
641 * tracepoint_probe_update_all() must be called
642 * before the module containing tp_cb is unloaded.
644 //ust// module_put(__module_text_address((unsigned long)elem->tp_cb));
647 elem
->single
.func
= __ust_marker_empty_function
;
648 /* Update the function before setting the ptype */
650 elem
->ptype
= 0; /* single probe */
652 * Leave the private data and channel_id/event_id there, because removal
653 * is racy and should be done only after an RCU period. These are never
654 * used until the next initialization anyway.
659 * is_ust_marker_enabled - Check if a ust_marker is enabled
660 * @channel: channel name
661 * @name: ust_marker name
663 * Returns 1 if the ust_marker is enabled, 0 if disabled.
665 int is_ust_marker_enabled(const char *channel
, const char *name
)
667 struct ust_marker_entry
*entry
;
669 pthread_mutex_lock(&ust_marker_mutex
);
670 entry
= get_ust_marker(channel
, name
);
671 pthread_mutex_unlock(&ust_marker_mutex
);
673 return entry
&& !!entry
->refcount
;
677 * ust_marker_update_probe_range - Update a probe range
678 * @begin: beginning of the range
679 * @end: end of the range
681 * Updates the probe callback corresponding to a range of ust_marker.
683 void ust_marker_update_probe_range(struct ust_marker
* const *begin
,
684 struct ust_marker
* const *end
)
686 struct ust_marker
* const *iter
;
687 struct ust_marker_entry
*mark_entry
;
689 pthread_mutex_lock(&ust_marker_mutex
);
690 for (iter
= begin
; iter
< end
; iter
++) {
692 continue; /* skip dummy */
693 mark_entry
= get_ust_marker((*iter
)->channel
, (*iter
)->name
);
695 set_ust_marker(mark_entry
, *iter
, !!mark_entry
->refcount
);
697 * ignore error, continue
700 disable_ust_marker(*iter
);
703 pthread_mutex_unlock(&ust_marker_mutex
);
706 static void lib_update_ust_marker(void)
708 struct ust_marker_lib
*lib
;
710 /* FIXME: we should probably take a mutex here on libs */
711 //ust// pthread_mutex_lock(&module_mutex);
712 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
)
713 ust_marker_update_probe_range(lib
->ust_marker_start
,
714 lib
->ust_marker_start
+ lib
->ust_marker_count
);
715 //ust// pthread_mutex_unlock(&module_mutex);
719 * Update probes, removing the faulty probes.
721 * Internal callback only changed before the first probe is connected to it.
722 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
723 * transitions. All other transitions will leave the old private data valid.
724 * This makes the non-atomicity of the callback/private data updates valid.
726 * "special case" updates :
731 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
732 * Site effect : ust_marker_set_format may delete the ust_marker entry (creating a
735 static void ust_marker_update_probes(void)
737 lib_update_ust_marker();
738 tracepoint_probe_update_all();
739 ust_marker_update_processes();
743 * ust_marker_probe_register - Connect a probe to a ust_marker
744 * @channel: ust_marker channel
745 * @name: ust_marker name
746 * @format: format string
747 * @probe: probe handler
748 * @probe_private: probe private data
750 * private data must be a valid allocated memory address, or NULL.
751 * Returns 0 if ok, error value on error.
752 * The probe address must at least be aligned on the architecture pointer size.
754 int ust_marker_probe_register(const char *channel
, const char *name
,
755 const char *format
, ust_marker_probe_func
*probe
,
758 struct ust_marker_entry
*entry
;
759 int ret
= 0, ret_err
;
760 struct ust_marker_probe_closure
*old
;
763 pthread_mutex_lock(&ust_marker_mutex
);
764 entry
= get_ust_marker(channel
, name
);
767 entry
= add_ust_marker(channel
, name
, format
);
769 ret
= PTR_ERR(entry
);
772 ret
= ltt_channels_register(channel
);
774 goto error_remove_ust_marker
;
775 ret
= ltt_channels_get_index_from_name(channel
);
777 goto error_unregister_channel
;
778 entry
->channel_id
= ret
;
779 ret
= ltt_channels_get_event_id(channel
, name
);
781 goto error_unregister_channel
;
782 entry
->event_id
= ret
;
784 __ust_marker(metadata
, core_marker_id
, NULL
,
785 "channel %s name %s event_id %hu "
786 "int #1u%zu long #1u%zu pointer #1u%zu "
787 "size_t #1u%zu alignment #1u%u",
788 channel
, name
, entry
->event_id
,
789 sizeof(int), sizeof(long), sizeof(void *),
790 sizeof(size_t), ltt_get_alignment());
793 ret
= ust_marker_set_format(entry
, format
);
794 else if (strcmp(entry
->format
, format
))
801 * If we detect that a call_rcu is pending for this ust_marker,
802 * make sure it's executed now.
804 //ust// if (entry->rcu_pending)
805 //ust// rcu_cmm_barrier_sched();
806 old
= ust_marker_entry_add_probe(entry
, probe
, probe_private
);
810 goto error_unregister_channel
;
814 pthread_mutex_unlock(&ust_marker_mutex
);
816 /* Activate ust_marker if necessary */
817 ust_marker_update_probes();
819 pthread_mutex_lock(&ust_marker_mutex
);
820 entry
= get_ust_marker(channel
, name
);
823 //ust// if (entry->rcu_pending)
824 //ust// rcu_cmm_barrier_sched();
826 entry
->rcu_pending
= 1;
827 /* write rcu_pending before calling the RCU callback */
829 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
830 synchronize_rcu(); free_old_closure(&entry
->rcu
);
833 error_unregister_channel
:
834 ret_err
= ltt_channels_unregister(channel
);
836 error_remove_ust_marker
:
837 ret_err
= remove_ust_marker(channel
, name
);
840 pthread_mutex_unlock(&ust_marker_mutex
);
843 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_register);
846 * ust_marker_probe_unregister - Disconnect a probe from a ust_marker
847 * @channel: ust_marker channel
848 * @name: ust_marker name
849 * @probe: probe function pointer
850 * @probe_private: probe private data
852 * Returns the private data given to ust_marker_probe_register, or an ERR_PTR().
853 * We do not need to call a synchronize_sched to make sure the probes have
854 * finished running before doing a module unload, because the module unload
855 * itself uses stop_machine(), which insures that every preempt disabled section
858 int ust_marker_probe_unregister(const char *channel
, const char *name
,
859 ust_marker_probe_func
*probe
, void *probe_private
)
861 struct ust_marker_entry
*entry
;
862 struct ust_marker_probe_closure
*old
;
865 pthread_mutex_lock(&ust_marker_mutex
);
866 entry
= get_ust_marker(channel
, name
);
869 //ust// if (entry->rcu_pending)
870 //ust// rcu_cmm_barrier_sched();
871 old
= ust_marker_entry_remove_probe(entry
, probe
, probe_private
);
872 pthread_mutex_unlock(&ust_marker_mutex
);
874 ust_marker_update_probes();
876 pthread_mutex_lock(&ust_marker_mutex
);
877 entry
= get_ust_marker(channel
, name
);
880 //ust// if (entry->rcu_pending)
881 //ust// rcu_cmm_barrier_sched();
883 entry
->rcu_pending
= 1;
884 /* write rcu_pending before calling the RCU callback */
886 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
887 synchronize_rcu(); free_old_closure(&entry
->rcu
);
888 remove_ust_marker(channel
, name
); /* Ignore busy error message */
891 pthread_mutex_unlock(&ust_marker_mutex
);
894 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_unregister);
896 static struct ust_marker_entry
*
897 get_ust_marker_from_private_data(ust_marker_probe_func
*probe
, void *probe_private
)
899 struct ust_marker_entry
*entry
;
901 struct cds_hlist_head
*head
;
902 struct cds_hlist_node
*node
;
904 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
905 head
= &ust_marker_table
[i
];
906 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
908 if (entry
->single
.func
== probe
909 && entry
->single
.probe_private
913 struct ust_marker_probe_closure
*closure
;
914 closure
= entry
->multi
;
915 for (i
= 0; closure
[i
].func
; i
++) {
916 if (closure
[i
].func
== probe
&&
917 closure
[i
].probe_private
928 * ust_marker_probe_unregister_private_data - Disconnect a probe from a ust_marker
929 * @probe: probe function
930 * @probe_private: probe private data
932 * Unregister a probe by providing the registered private data.
933 * Only removes the first ust_marker found in hash table.
934 * Return 0 on success or error value.
935 * We do not need to call a synchronize_sched to make sure the probes have
936 * finished running before doing a module unload, because the module unload
937 * itself uses stop_machine(), which insures that every preempt disabled section
940 int ust_marker_probe_unregister_private_data(ust_marker_probe_func
*probe
,
943 struct ust_marker_entry
*entry
;
945 struct ust_marker_probe_closure
*old
;
946 char *channel
= NULL
, *name
= NULL
;
948 pthread_mutex_lock(&ust_marker_mutex
);
949 entry
= get_ust_marker_from_private_data(probe
, probe_private
);
954 //ust// if (entry->rcu_pending)
955 //ust// rcu_cmm_barrier_sched();
956 old
= ust_marker_entry_remove_probe(entry
, NULL
, probe_private
);
957 channel
= strdup(entry
->channel
);
958 name
= strdup(entry
->name
);
959 pthread_mutex_unlock(&ust_marker_mutex
);
961 ust_marker_update_probes();
963 pthread_mutex_lock(&ust_marker_mutex
);
964 entry
= get_ust_marker(channel
, name
);
967 //ust// if (entry->rcu_pending)
968 //ust// rcu_cmm_barrier_sched();
970 entry
->rcu_pending
= 1;
971 /* write rcu_pending before calling the RCU callback */
973 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
974 synchronize_rcu(); free_old_closure(&entry
->rcu
);
975 /* Ignore busy error message */
976 remove_ust_marker(channel
, name
);
978 pthread_mutex_unlock(&ust_marker_mutex
);
983 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_unregister_private_data);
986 * ust_marker_get_private_data - Get a ust_marker's probe private data
987 * @channel: ust_marker channel
988 * @name: ust_marker name
989 * @probe: probe to match
990 * @num: get the nth matching probe's private data
992 * Returns the nth private data pointer (starting from 0) matching, or an
994 * Returns the private data pointer, or an ERR_PTR.
995 * The private data pointer should _only_ be dereferenced if the caller is the
996 * owner of the data, or its content could vanish. This is mostly used to
997 * confirm that a caller is the owner of a registered probe.
999 void *ust_marker_get_private_data(const char *channel
, const char *name
,
1000 ust_marker_probe_func
*probe
, int num
)
1002 struct cds_hlist_head
*head
;
1003 struct cds_hlist_node
*node
;
1004 struct ust_marker_entry
*e
;
1005 size_t channel_len
= strlen(channel
) + 1;
1006 size_t name_len
= strlen(name
) + 1;
1010 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
1011 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
1012 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
1013 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
1015 if (num
== 0 && e
->single
.func
== probe
)
1016 return e
->single
.probe_private
;
1018 struct ust_marker_probe_closure
*closure
;
1021 for (i
= 0; closure
[i
].func
; i
++) {
1022 if (closure
[i
].func
!= probe
)
1025 return closure
[i
].probe_private
;
1031 return ERR_PTR(-ENOENT
);
1033 //ust// EXPORT_SYMBOL_GPL(ust_marker_get_private_data);
1036 * ust_marker_compact_event_ids - Compact ust_marker event IDs and reassign channels
1038 * Called when no channel users are active by the channel infrastructure.
1039 * Called with lock_ust_marker() and channel mutex held.
1041 //ust// void ust_marker_compact_event_ids(void)
1043 //ust// struct ust_marker_entry *entry;
1044 //ust// unsigned int i;
1045 //ust// struct hlist_head *head;
1046 //ust// struct hlist_node *node;
1049 //ust// for (i = 0; i < UST_MARKER_TABLE_SIZE; i++) {
1050 //ust// head = &ust_marker_table[i];
1051 //ust// hlist_for_each_entry(entry, node, head, hlist) {
1052 //ust// ret = ltt_channels_get_index_from_name(entry->channel);
1053 //ust// WARN_ON(ret < 0);
1054 //ust// entry->channel_id = ret;
1055 //ust// ret = _ltt_channels_get_event_id(entry->channel,
1056 //ust// entry->name);
1057 //ust// WARN_ON(ret < 0);
1058 //ust// entry->event_id = ret;
1063 //ust//#ifdef CONFIG_MODULES
1066 * Returns 0 if current not found.
1067 * Returns 1 if current found.
1069 int lib_get_iter_ust_marker(struct ust_marker_iter
*iter
)
1071 struct ust_marker_lib
*iter_lib
;
1074 //ust// pthread_mutex_lock(&module_mutex);
1075 cds_list_for_each_entry(iter_lib
, &ust_marker_libs
, list
) {
1076 if (iter_lib
< iter
->lib
)
1078 else if (iter_lib
> iter
->lib
)
1079 iter
->ust_marker
= NULL
;
1080 found
= ust_marker_get_iter_range(&iter
->ust_marker
,
1081 iter_lib
->ust_marker_start
,
1082 iter_lib
->ust_marker_start
+ iter_lib
->ust_marker_count
);
1084 iter
->lib
= iter_lib
;
1088 //ust// pthread_mutex_unlock(&module_mutex);
1093 * ust_marker_get_iter_range - Get a next ust_marker iterator given a range.
1094 * @ust_marker: current ust_marker (in), next ust_marker (out)
1095 * @begin: beginning of the range
1096 * @end: end of the range
1098 * Returns whether a next ust_marker has been found (1) or not (0).
1099 * Will return the first ust_marker in the range if the input ust_marker is NULL.
1101 int ust_marker_get_iter_range(struct ust_marker
* const **ust_marker
,
1102 struct ust_marker
* const *begin
,
1103 struct ust_marker
* const *end
)
1105 if (!*ust_marker
&& begin
!= end
)
1106 *ust_marker
= begin
;
1107 while (*ust_marker
>= begin
&& *ust_marker
< end
) {
1109 (*ust_marker
)++; /* skip dummy */
1115 //ust// EXPORT_SYMBOL_GPL(ust_marker_get_iter_range);
1117 static void ust_marker_get_iter(struct ust_marker_iter
*iter
)
1121 found
= lib_get_iter_ust_marker(iter
);
1123 ust_marker_iter_reset(iter
);
1126 void ust_marker_iter_start(struct ust_marker_iter
*iter
)
1128 ust_marker_get_iter(iter
);
1130 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_start);
1132 void ust_marker_iter_next(struct ust_marker_iter
*iter
)
1136 * iter->ust_marker may be invalid because we blindly incremented it.
1137 * Make sure it is valid by marshalling on the ust_marker, getting the
1138 * ust_marker from following modules if necessary.
1140 ust_marker_get_iter(iter
);
1142 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_next);
1144 void ust_marker_iter_stop(struct ust_marker_iter
*iter
)
1147 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_stop);
1149 void ust_marker_iter_reset(struct ust_marker_iter
*iter
)
1152 iter
->ust_marker
= NULL
;
1154 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_reset);
1156 #ifdef CONFIG_UST_MARKER_USERSPACE
1158 * must be called with current->user_ust_marker_mutex held
1160 static void free_user_ust_marker(char __user
*state
, struct cds_hlist_head
*head
)
1162 struct user_ust_marker
*umark
;
1163 struct cds_hlist_node
*pos
, *n
;
1165 cds_hlist_for_each_entry_safe(umark
, pos
, n
, head
, hlist
) {
1166 if (umark
->state
== state
) {
1167 cds_hlist_del(&umark
->hlist
);
1174 * Update current process.
1175 * Note that we have to wait a whole scheduler period before we are sure that
1176 * every running userspace threads have their ust_marker updated.
1177 * (synchronize_sched() can be used to insure this).
1179 //ust// void ust_marker_update_process(void)
1181 //ust// struct user_ust_marker *umark;
1182 //ust// struct hlist_node *pos;
1183 //ust// struct ust_marker_entry *entry;
1185 //ust// pthread_mutex_lock(&ust_marker_mutex);
1186 //ust// pthread_mutex_lock(¤t->group_leader->user_ust_marker_mutex);
1187 //ust// if (strcmp(current->comm, "testprog") == 0)
1188 //ust// DBG("do update pending for testprog");
1189 //ust// hlist_for_each_entry(umark, pos,
1190 //ust// ¤t->group_leader->user_ust_marker, hlist) {
1191 //ust// DBG("Updating ust_marker %s in %s", umark->name, current->comm);
1192 //ust// entry = get_ust_marker("userspace", umark->name);
1193 //ust// if (entry) {
1194 //ust// if (entry->format &&
1195 //ust// strcmp(entry->format, umark->format) != 0) {
1196 //ust// WARN("error, wrong format in process %s",
1197 //ust// current->comm);
1200 //ust// if (put_user(!!entry->refcount, umark->state)) {
1201 //ust// WARN("ust_marker in %s caused a fault",
1202 //ust// current->comm);
1206 //ust// if (put_user(0, umark->state)) {
1207 //ust// WARN("ust_marker in %s caused a fault", current->comm);
1212 //ust// clear_thread_flag(TIF_UST_MARKER_PENDING);
1213 //ust// pthread_mutex_unlock(¤t->group_leader->user_ust_marker_mutex);
1214 //ust// pthread_mutex_unlock(&ust_marker_mutex);
1218 * Called at process exit and upon do_execve().
1219 * We assume that when the leader exits, no more references can be done to the
1220 * leader structure by the other threads.
1222 void exit_user_ust_marker(struct task_struct
*p
)
1224 struct user_ust_marker
*umark
;
1225 struct cds_hlist_node
*pos
, *n
;
1227 if (thread_group_leader(p
)) {
1228 pthread_mutex_lock(&ust_marker_mutex
);
1229 pthread_mutex_lock(&p
->user_ust_marker_mutex
);
1230 cds_hlist_for_each_entry_safe(umark
, pos
, n
, &p
->user_ust_marker
,
1233 INIT_HLIST_HEAD(&p
->user_ust_marker
);
1234 p
->user_ust_marker_sequence
++;
1235 pthread_mutex_unlock(&p
->user_ust_marker_mutex
);
1236 pthread_mutex_unlock(&ust_marker_mutex
);
1240 int is_ust_marker_enabled(const char *channel
, const char *name
)
1242 struct ust_marker_entry
*entry
;
1244 pthread_mutex_lock(&ust_marker_mutex
);
1245 entry
= get_ust_marker(channel
, name
);
1246 pthread_mutex_unlock(&ust_marker_mutex
);
1248 return entry
&& !!entry
->refcount
;
1252 int ust_marker_module_notify(struct notifier_block
*self
,
1253 unsigned long val
, void *data
)
1255 struct module
*mod
= data
;
1258 case MODULE_STATE_COMING
:
1259 ust_marker_update_probe_range(mod
->ust_marker
,
1260 mod
->ust_marker
+ mod
->num_ust_marker
);
1262 case MODULE_STATE_GOING
:
1263 ust_marker_update_probe_range(mod
->ust_marker
,
1264 mod
->ust_marker
+ mod
->num_ust_marker
);
1270 struct notifier_block ust_marker_module_nb
= {
1271 .notifier_call
= ust_marker_module_notify
,
1275 //ust// static int init_ust_marker(void)
1277 //ust// return register_module_notifier(&ust_marker_module_nb);
1279 //ust// __initcall(init_ust_marker);
1280 /* TODO: call ust_marker_module_nb() when a library is linked at runtime (dlopen)? */
1282 #endif /* CONFIG_MODULES */
1284 void ltt_dump_ust_marker_state(struct ust_trace
*trace
)
1286 struct ust_marker_entry
*entry
;
1287 struct ltt_probe_private_data call_data
;
1288 struct cds_hlist_head
*head
;
1289 struct cds_hlist_node
*node
;
1292 pthread_mutex_lock(&ust_marker_mutex
);
1293 call_data
.trace
= trace
;
1294 call_data
.serializer
= NULL
;
1296 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
1297 head
= &ust_marker_table
[i
];
1298 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
1299 __ust_marker(metadata
, core_marker_id
,
1301 "channel %s name %s event_id %hu "
1302 "int #1u%zu long #1u%zu pointer #1u%zu "
1303 "size_t #1u%zu alignment #1u%u",
1307 sizeof(int), sizeof(long),
1308 sizeof(void *), sizeof(size_t),
1309 ltt_get_alignment());
1311 __ust_marker(metadata
,
1314 "channel %s name %s format %s",
1320 pthread_mutex_unlock(&ust_marker_mutex
);
1322 //ust// EXPORT_SYMBOL_GPL(ltt_dump_ust_marker_state);
1324 static void (*new_ust_marker_cb
)(struct ust_marker
*) = NULL
;
1326 void ust_marker_set_new_ust_marker_cb(void (*cb
)(struct ust_marker
*))
1328 new_ust_marker_cb
= cb
;
1331 static void new_ust_marker(struct ust_marker
* const *start
, struct ust_marker
* const *end
)
1333 if (new_ust_marker_cb
) {
1334 struct ust_marker
* const *m
;
1336 for(m
= start
; m
< end
; m
++) {
1338 new_ust_marker_cb(*m
);
1343 int ust_marker_register_lib(struct ust_marker
* const *ust_marker_start
, int ust_marker_count
)
1345 struct ust_marker_lib
*pl
, *iter
;
1347 pl
= (struct ust_marker_lib
*) zmalloc(sizeof(struct ust_marker_lib
));
1349 pl
->ust_marker_start
= ust_marker_start
;
1350 pl
->ust_marker_count
= ust_marker_count
;
1352 /* FIXME: maybe protect this with its own mutex? */
1356 * We sort the libs by struct lib pointer address.
1358 cds_list_for_each_entry_reverse(iter
, &ust_marker_libs
, list
) {
1359 BUG_ON(iter
== pl
); /* Should never be in the list twice */
1361 /* We belong to the location right after iter. */
1362 cds_list_add(&pl
->list
, &iter
->list
);
1366 /* We should be added at the head of the list */
1367 cds_list_add(&pl
->list
, &ust_marker_libs
);
1369 unlock_ust_marker();
1371 new_ust_marker(ust_marker_start
, ust_marker_start
+ ust_marker_count
);
1373 /* FIXME: update just the loaded lib */
1374 lib_update_ust_marker();
1376 DBG("just registered a ust_marker section from %p and having %d ust_marker (minus dummy ust_marker)", ust_marker_start
, ust_marker_count
);
1381 int ust_marker_unregister_lib(struct ust_marker
* const *ust_marker_start
)
1383 struct ust_marker_lib
*lib
;
1385 /*FIXME: implement; but before implementing, ust_marker_register_lib must
1386 have appropriate locking. */
1390 /* FIXME: we should probably take a mutex here on libs */
1391 //ust// pthread_mutex_lock(&module_mutex);
1392 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
) {
1393 if(lib
->ust_marker_start
== ust_marker_start
) {
1394 struct ust_marker_lib
*lib2free
= lib
;
1395 cds_list_del(&lib
->list
);
1401 unlock_ust_marker();
1406 static int initialized
= 0;
1408 void __attribute__((constructor
)) init_ust_marker(void)
1411 ust_marker_register_lib(__start___ust_marker_ptrs
,
1412 __stop___ust_marker_ptrs
1413 - __start___ust_marker_ptrs
);
1418 void __attribute__((destructor
)) destroy_ust_marker(void)
1420 ust_marker_unregister_lib(__start___ust_marker_ptrs
);