2 * Copyright (C) 2007 Mathieu Desnoyers
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 #include <linux/module.h>
19 #include <linux/mutex.h>
20 #include <linux/types.h>
21 #include <linux/jhash.h>
22 #include <linux/list.h>
23 #include <linux/rcupdate.h>
24 #include <linux/marker.h>
25 #include <linux/err.h>
26 #include <linux/slab.h>
27 #include <linux/immediate.h>
28 #include <linux/sched.h>
29 #include <linux/uaccess.h>
30 #include <linux/user_marker.h>
31 #include <linux/ltt-tracer.h>
33 extern struct marker __start___markers
[];
34 extern struct marker __stop___markers
[];
36 /* Set to 1 to enable marker debug output */
37 static const int marker_debug
;
40 * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
41 * and module markers and the hash table.
43 static DEFINE_MUTEX(markers_mutex
);
45 void lock_markers(void)
47 mutex_lock(&markers_mutex
);
50 void unlock_markers(void)
52 mutex_unlock(&markers_mutex
);
56 * Marker hash table, containing the active markers.
57 * Protected by module_mutex.
59 #define MARKER_HASH_BITS 6
60 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
61 static struct hlist_head marker_table
[MARKER_TABLE_SIZE
];
65 * It is used to make sure every handler has finished using its private data
66 * between two consecutive operation (add or remove) on a given marker. It is
67 * also used to delay the free of multiple probes array until a quiescent state
69 * marker entries modifications are protected by the markers_mutex.
72 struct hlist_node hlist
;
76 void (*call
)(const struct marker
*mdata
, void *call_private
, ...);
77 struct marker_probe_closure single
;
78 struct marker_probe_closure
*multi
;
79 int refcount
; /* Number of times armed. 0 if disarmed. */
85 unsigned char ptype
:1;
86 unsigned char format_allocated
:1;
87 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
90 #ifdef CONFIG_MARKERS_USERSPACE
91 static void marker_update_processes(void);
93 static void marker_update_processes(void)
99 * __mark_empty_function - Empty probe callback
100 * @mdata: marker data
101 * @probe_private: probe private data
102 * @call_private: call site private data
103 * @fmt: format string
104 * @...: variable argument list
106 * Empty callback provided as a probe to the markers. By providing this to a
107 * disabled marker, we make sure the execution flow is always valid even
108 * though the function pointer change and the marker enabling are two distinct
109 * operations that modifies the execution flow of preemptible code.
111 notrace
void __mark_empty_function(const struct marker
*mdata
,
112 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
115 EXPORT_SYMBOL_GPL(__mark_empty_function
);
118 * marker_probe_cb Callback that prepares the variable argument list for probes.
119 * @mdata: pointer of type struct marker
120 * @call_private: caller site private data
121 * @...: Variable argument list.
123 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
124 * need to put a full smp_rmb() in this branch. This is why we do not use
125 * rcu_dereference() for the pointer read.
127 notrace
void marker_probe_cb(const struct marker
*mdata
,
128 void *call_private
, ...)
134 * rcu_read_lock_sched does two things : disabling preemption to make
135 * sure the teardown of the callbacks can be done correctly when they
136 * are in modules and they insure RCU read coherency.
138 rcu_read_lock_sched_notrace();
139 ptype
= mdata
->ptype
;
140 if (likely(!ptype
)) {
141 marker_probe_func
*func
;
142 /* Must read the ptype before ptr. They are not data dependant,
143 * so we put an explicit smp_rmb() here. */
145 func
= mdata
->single
.func
;
146 /* Must read the ptr before private data. They are not data
147 * dependant, so we put an explicit smp_rmb() here. */
149 va_start(args
, call_private
);
150 func(mdata
, mdata
->single
.probe_private
, call_private
,
151 mdata
->format
, &args
);
154 struct marker_probe_closure
*multi
;
157 * Read mdata->ptype before mdata->multi.
160 multi
= mdata
->multi
;
162 * multi points to an array, therefore accessing the array
163 * depends on reading multi. However, even in this case,
164 * we must insure that the pointer is read _before_ the array
165 * data. Same as rcu_dereference, but we need a full smp_rmb()
166 * in the fast path, so put the explicit barrier here.
168 smp_read_barrier_depends();
169 for (i
= 0; multi
[i
].func
; i
++) {
170 va_start(args
, call_private
);
171 multi
[i
].func(mdata
, multi
[i
].probe_private
,
172 call_private
, mdata
->format
, &args
);
176 rcu_read_unlock_sched_notrace();
178 EXPORT_SYMBOL_GPL(marker_probe_cb
);
181 * marker_probe_cb Callback that does not prepare the variable argument list.
182 * @mdata: pointer of type struct marker
183 * @call_private: caller site private data
184 * @...: Variable argument list.
186 * Should be connected to markers "MARK_NOARGS".
188 static notrace
void marker_probe_cb_noarg(const struct marker
*mdata
,
189 void *call_private
, ...)
191 va_list args
; /* not initialized */
194 rcu_read_lock_sched_notrace();
195 ptype
= mdata
->ptype
;
196 if (likely(!ptype
)) {
197 marker_probe_func
*func
;
198 /* Must read the ptype before ptr. They are not data dependant,
199 * so we put an explicit smp_rmb() here. */
201 func
= mdata
->single
.func
;
202 /* Must read the ptr before private data. They are not data
203 * dependant, so we put an explicit smp_rmb() here. */
205 func(mdata
, mdata
->single
.probe_private
, call_private
,
206 mdata
->format
, &args
);
208 struct marker_probe_closure
*multi
;
211 * Read mdata->ptype before mdata->multi.
214 multi
= mdata
->multi
;
216 * multi points to an array, therefore accessing the array
217 * depends on reading multi. However, even in this case,
218 * we must insure that the pointer is read _before_ the array
219 * data. Same as rcu_dereference, but we need a full smp_rmb()
220 * in the fast path, so put the explicit barrier here.
222 smp_read_barrier_depends();
223 for (i
= 0; multi
[i
].func
; i
++)
224 multi
[i
].func(mdata
, multi
[i
].probe_private
,
225 call_private
, mdata
->format
, &args
);
227 rcu_read_unlock_sched_notrace();
230 static void free_old_closure(struct rcu_head
*head
)
232 struct marker_entry
*entry
= container_of(head
,
233 struct marker_entry
, rcu
);
234 kfree(entry
->oldptr
);
235 /* Make sure we free the data before setting the pending flag to 0 */
237 entry
->rcu_pending
= 0;
240 static void debug_print_probes(struct marker_entry
*entry
)
248 printk(KERN_DEBUG
"Single probe : %p %p\n",
250 entry
->single
.probe_private
);
252 for (i
= 0; entry
->multi
[i
].func
; i
++)
253 printk(KERN_DEBUG
"Multi probe %d : %p %p\n", i
,
254 entry
->multi
[i
].func
,
255 entry
->multi
[i
].probe_private
);
259 static struct marker_probe_closure
*
260 marker_entry_add_probe(struct marker_entry
*entry
,
261 marker_probe_func
*probe
, void *probe_private
)
264 struct marker_probe_closure
*old
, *new;
268 debug_print_probes(entry
);
271 if (entry
->single
.func
== probe
&&
272 entry
->single
.probe_private
== probe_private
)
273 return ERR_PTR(-EBUSY
);
274 if (entry
->single
.func
== __mark_empty_function
) {
276 entry
->single
.func
= probe
;
277 entry
->single
.probe_private
= probe_private
;
280 debug_print_probes(entry
);
288 /* (N -> N+1), (N != 0, 1) probes */
289 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++)
290 if (old
[nr_probes
].func
== probe
291 && old
[nr_probes
].probe_private
293 return ERR_PTR(-EBUSY
);
295 /* + 2 : one for new probe, one for NULL func */
296 new = kzalloc((nr_probes
+ 2) * sizeof(struct marker_probe_closure
),
299 return ERR_PTR(-ENOMEM
);
301 new[0] = entry
->single
;
304 nr_probes
* sizeof(struct marker_probe_closure
));
305 new[nr_probes
].func
= probe
;
306 new[nr_probes
].probe_private
= probe_private
;
307 entry
->refcount
= nr_probes
+ 1;
310 debug_print_probes(entry
);
314 static struct marker_probe_closure
*
315 marker_entry_remove_probe(struct marker_entry
*entry
,
316 marker_probe_func
*probe
, void *probe_private
)
318 int nr_probes
= 0, nr_del
= 0, i
;
319 struct marker_probe_closure
*old
, *new;
323 debug_print_probes(entry
);
325 /* 0 -> N is an error */
326 WARN_ON(entry
->single
.func
== __mark_empty_function
);
328 WARN_ON(probe
&& entry
->single
.func
!= probe
);
329 WARN_ON(entry
->single
.probe_private
!= probe_private
);
330 entry
->single
.func
= __mark_empty_function
;
333 debug_print_probes(entry
);
336 /* (N -> M), (N > 1, M >= 0) probes */
337 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
338 if ((!probe
|| old
[nr_probes
].func
== probe
)
339 && old
[nr_probes
].probe_private
345 if (nr_probes
- nr_del
== 0) {
346 /* N -> 0, (N > 1) */
347 entry
->single
.func
= __mark_empty_function
;
350 } else if (nr_probes
- nr_del
== 1) {
351 /* N -> 1, (N > 1) */
352 for (i
= 0; old
[i
].func
; i
++)
353 if ((probe
&& old
[i
].func
!= probe
) ||
354 old
[i
].probe_private
!= probe_private
)
355 entry
->single
= old
[i
];
360 /* N -> M, (N > 1, M > 1) */
362 new = kzalloc((nr_probes
- nr_del
+ 1)
363 * sizeof(struct marker_probe_closure
), GFP_KERNEL
);
365 return ERR_PTR(-ENOMEM
);
366 for (i
= 0; old
[i
].func
; i
++)
367 if ((probe
&& old
[i
].func
!= probe
) ||
368 old
[i
].probe_private
!= probe_private
)
370 entry
->refcount
= nr_probes
- nr_del
;
374 debug_print_probes(entry
);
379 * Get marker if the marker is present in the marker hash table.
380 * Must be called with markers_mutex held.
381 * Returns NULL if not present.
383 static struct marker_entry
*get_marker(const char *channel
, const char *name
)
385 struct hlist_head
*head
;
386 struct hlist_node
*node
;
387 struct marker_entry
*e
;
388 size_t channel_len
= strlen(channel
) + 1;
389 size_t name_len
= strlen(name
) + 1;
392 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
393 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
394 hlist_for_each_entry(e
, node
, head
, hlist
) {
395 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
402 * Add the marker to the marker hash table. Must be called with markers_mutex
405 static struct marker_entry
*add_marker(const char *channel
, const char *name
,
408 struct hlist_head
*head
;
409 struct hlist_node
*node
;
410 struct marker_entry
*e
;
411 size_t channel_len
= strlen(channel
) + 1;
412 size_t name_len
= strlen(name
) + 1;
413 size_t format_len
= 0;
416 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
418 format_len
= strlen(format
) + 1;
419 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
420 hlist_for_each_entry(e
, node
, head
, hlist
) {
421 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
423 "Marker %s.%s busy\n", channel
, name
);
424 return ERR_PTR(-EBUSY
); /* Already there */
428 * Using kmalloc here to allocate a variable length element. Could
429 * cause some memory fragmentation if overused.
431 e
= kmalloc(sizeof(struct marker_entry
)
432 + channel_len
+ name_len
+ format_len
,
435 return ERR_PTR(-ENOMEM
);
436 memcpy(e
->channel
, channel
, channel_len
);
437 e
->name
= &e
->channel
[channel_len
];
438 memcpy(e
->name
, name
, name_len
);
440 e
->format
= &e
->name
[channel_len
+ name_len
];
441 memcpy(e
->format
, format
, format_len
);
442 if (strcmp(e
->format
, MARK_NOARGS
) == 0)
443 e
->call
= marker_probe_cb_noarg
;
445 e
->call
= marker_probe_cb
;
446 trace_mark(metadata
, core_marker_format
,
447 "channel %s name %s format %s",
448 e
->channel
, e
->name
, e
->format
);
451 e
->call
= marker_probe_cb
;
453 e
->single
.func
= __mark_empty_function
;
454 e
->single
.probe_private
= NULL
;
457 e
->format_allocated
= 0;
460 hlist_add_head(&e
->hlist
, head
);
465 * Remove the marker from the marker hash table. Must be called with mutex_lock
468 static int remove_marker(const char *channel
, const char *name
)
470 struct hlist_head
*head
;
471 struct hlist_node
*node
;
472 struct marker_entry
*e
;
474 size_t channel_len
= strlen(channel
) + 1;
475 size_t name_len
= strlen(name
) + 1;
479 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
480 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
481 hlist_for_each_entry(e
, node
, head
, hlist
) {
482 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
489 if (e
->single
.func
!= __mark_empty_function
)
491 hlist_del(&e
->hlist
);
492 if (e
->format_allocated
)
494 ret
= ltt_channels_unregister(e
->channel
);
496 /* Make sure the call_rcu has been executed */
504 * Set the mark_entry format to the format found in the element.
506 static int marker_set_format(struct marker_entry
*entry
, const char *format
)
508 entry
->format
= kstrdup(format
, GFP_KERNEL
);
511 entry
->format_allocated
= 1;
513 trace_mark(metadata
, core_marker_format
,
514 "channel %s name %s format %s",
515 entry
->channel
, entry
->name
, entry
->format
);
520 * Sets the probe callback corresponding to one marker.
522 static int set_marker(struct marker_entry
*entry
, struct marker
*elem
,
526 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
529 if (strcmp(entry
->format
, elem
->format
) != 0) {
531 "Format mismatch for probe %s "
532 "(%s), marker (%s)\n",
539 ret
= marker_set_format(entry
, elem
->format
);
545 * probe_cb setup (statically known) is done here. It is
546 * asynchronous with the rest of execution, therefore we only
547 * pass from a "safe" callback (with argument) to an "unsafe"
548 * callback (does not set arguments).
550 elem
->call
= entry
->call
;
551 elem
->channel_id
= entry
->channel_id
;
552 elem
->event_id
= entry
->event_id
;
555 * We only update the single probe private data when the ptr is
556 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
558 WARN_ON(elem
->single
.func
!= __mark_empty_function
559 && elem
->single
.probe_private
!= entry
->single
.probe_private
561 elem
->single
.probe_private
= entry
->single
.probe_private
;
563 * Make sure the private data is valid when we update the
567 elem
->single
.func
= entry
->single
.func
;
569 * We also make sure that the new probe callbacks array is consistent
570 * before setting a pointer to it.
572 rcu_assign_pointer(elem
->multi
, entry
->multi
);
574 * Update the function or multi probe array pointer before setting the
578 elem
->ptype
= entry
->ptype
;
580 if (elem
->tp_name
&& (active
^ _imv_read(elem
->state
))) {
581 WARN_ON(!elem
->tp_cb
);
583 * It is ok to directly call the probe registration because type
584 * checking has been done in the __trace_mark_tp() macro.
589 * try_module_get should always succeed because we hold
590 * markers_mutex to get the tp_cb address.
592 ret
= try_module_get(__module_text_address(
593 (unsigned long)elem
->tp_cb
));
595 ret
= tracepoint_probe_register_noupdate(
599 ret
= tracepoint_probe_unregister_noupdate(
603 * tracepoint_probe_update_all() must be called
604 * before the module containing tp_cb is unloaded.
606 module_put(__module_text_address(
607 (unsigned long)elem
->tp_cb
));
610 elem
->state__imv
= active
;
616 * Disable a marker and its probe callback.
617 * Note: only waiting an RCU period after setting elem->call to the empty
618 * function insures that the original callback is not used anymore. This insured
619 * by rcu_read_lock_sched around the call site.
621 static void disable_marker(struct marker
*elem
)
625 /* leave "call" as is. It is known statically. */
626 if (elem
->tp_name
&& _imv_read(elem
->state
)) {
627 WARN_ON(!elem
->tp_cb
);
629 * It is ok to directly call the probe registration because type
630 * checking has been done in the __trace_mark_tp() macro.
632 ret
= tracepoint_probe_unregister_noupdate(elem
->tp_name
,
636 * tracepoint_probe_update_all() must be called
637 * before the module containing tp_cb is unloaded.
639 module_put(__module_text_address((unsigned long)elem
->tp_cb
));
641 elem
->state__imv
= 0;
642 elem
->single
.func
= __mark_empty_function
;
643 /* Update the function before setting the ptype */
645 elem
->ptype
= 0; /* single probe */
647 * Leave the private data and channel_id/event_id there, because removal
648 * is racy and should be done only after an RCU period. These are never
649 * used until the next initialization anyway.
654 * marker_update_probe_range - Update a probe range
655 * @begin: beginning of the range
656 * @end: end of the range
658 * Updates the probe callback corresponding to a range of markers.
660 void marker_update_probe_range(struct marker
*begin
,
664 struct marker_entry
*mark_entry
;
666 mutex_lock(&markers_mutex
);
667 for (iter
= begin
; iter
< end
; iter
++) {
668 mark_entry
= get_marker(iter
->channel
, iter
->name
);
670 set_marker(mark_entry
, iter
, !!mark_entry
->refcount
);
672 * ignore error, continue
675 disable_marker(iter
);
678 mutex_unlock(&markers_mutex
);
682 * Update probes, removing the faulty probes.
684 * Internal callback only changed before the first probe is connected to it.
685 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
686 * transitions. All other transitions will leave the old private data valid.
687 * This makes the non-atomicity of the callback/private data updates valid.
689 * "special case" updates :
694 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
695 * Site effect : marker_set_format may delete the marker entry (creating a
698 static void marker_update_probes(void)
700 /* Core kernel markers */
701 marker_update_probe_range(__start___markers
, __stop___markers
);
702 /* Markers in modules. */
703 module_update_markers();
704 tracepoint_probe_update_all();
705 /* Update immediate values */
708 marker_update_processes();
712 * marker_probe_register - Connect a probe to a marker
713 * @channel: marker channel
715 * @format: format string
716 * @probe: probe handler
717 * @probe_private: probe private data
719 * private data must be a valid allocated memory address, or NULL.
720 * Returns 0 if ok, error value on error.
721 * The probe address must at least be aligned on the architecture pointer size.
723 int marker_probe_register(const char *channel
, const char *name
,
724 const char *format
, marker_probe_func
*probe
,
727 struct marker_entry
*entry
;
728 int ret
= 0, ret_err
;
729 struct marker_probe_closure
*old
;
732 mutex_lock(&markers_mutex
);
733 entry
= get_marker(channel
, name
);
736 entry
= add_marker(channel
, name
, format
);
738 ret
= PTR_ERR(entry
);
741 ret
= ltt_channels_register(channel
);
743 goto error_remove_marker
;
744 ret
= ltt_channels_get_index_from_name(channel
);
746 goto error_unregister_channel
;
747 entry
->channel_id
= ret
;
748 ret
= ltt_channels_get_event_id(channel
, name
);
750 goto error_unregister_channel
;
751 entry
->event_id
= ret
;
753 trace_mark(metadata
, core_marker_id
,
754 "channel %s name %s event_id %hu "
755 "int #1u%zu long #1u%zu pointer #1u%zu "
756 "size_t #1u%zu alignment #1u%u",
757 channel
, name
, entry
->event_id
,
758 sizeof(int), sizeof(long), sizeof(void *),
759 sizeof(size_t), ltt_get_alignment());
762 ret
= marker_set_format(entry
, format
);
763 else if (strcmp(entry
->format
, format
))
770 * If we detect that a call_rcu is pending for this marker,
771 * make sure it's executed now.
773 if (entry
->rcu_pending
)
775 old
= marker_entry_add_probe(entry
, probe
, probe_private
);
779 goto error_unregister_channel
;
783 mutex_unlock(&markers_mutex
);
785 marker_update_probes();
787 mutex_lock(&markers_mutex
);
788 entry
= get_marker(channel
, name
);
791 if (entry
->rcu_pending
)
794 entry
->rcu_pending
= 1;
795 /* write rcu_pending before calling the RCU callback */
797 call_rcu_sched(&entry
->rcu
, free_old_closure
);
800 error_unregister_channel
:
801 ret_err
= ltt_channels_unregister(channel
);
804 ret_err
= remove_marker(channel
, name
);
807 mutex_unlock(&markers_mutex
);
810 EXPORT_SYMBOL_GPL(marker_probe_register
);
813 * marker_probe_unregister - Disconnect a probe from a marker
814 * @channel: marker channel
816 * @probe: probe function pointer
817 * @probe_private: probe private data
819 * Returns the private data given to marker_probe_register, or an ERR_PTR().
820 * We do not need to call a synchronize_sched to make sure the probes have
821 * finished running before doing a module unload, because the module unload
822 * itself uses stop_machine(), which insures that every preempt disabled section
825 int marker_probe_unregister(const char *channel
, const char *name
,
826 marker_probe_func
*probe
, void *probe_private
)
828 struct marker_entry
*entry
;
829 struct marker_probe_closure
*old
;
832 mutex_lock(&markers_mutex
);
833 entry
= get_marker(channel
, name
);
836 if (entry
->rcu_pending
)
838 old
= marker_entry_remove_probe(entry
, probe
, probe_private
);
839 mutex_unlock(&markers_mutex
);
841 marker_update_probes();
843 mutex_lock(&markers_mutex
);
844 entry
= get_marker(channel
, name
);
847 if (entry
->rcu_pending
)
850 entry
->rcu_pending
= 1;
851 /* write rcu_pending before calling the RCU callback */
853 call_rcu_sched(&entry
->rcu
, free_old_closure
);
854 remove_marker(channel
, name
); /* Ignore busy error message */
857 mutex_unlock(&markers_mutex
);
860 EXPORT_SYMBOL_GPL(marker_probe_unregister
);
862 static struct marker_entry
*
863 get_marker_from_private_data(marker_probe_func
*probe
, void *probe_private
)
865 struct marker_entry
*entry
;
867 struct hlist_head
*head
;
868 struct hlist_node
*node
;
870 for (i
= 0; i
< MARKER_TABLE_SIZE
; i
++) {
871 head
= &marker_table
[i
];
872 hlist_for_each_entry(entry
, node
, head
, hlist
) {
874 if (entry
->single
.func
== probe
875 && entry
->single
.probe_private
879 struct marker_probe_closure
*closure
;
880 closure
= entry
->multi
;
881 for (i
= 0; closure
[i
].func
; i
++) {
882 if (closure
[i
].func
== probe
&&
883 closure
[i
].probe_private
894 * marker_probe_unregister_private_data - Disconnect a probe from a marker
895 * @probe: probe function
896 * @probe_private: probe private data
898 * Unregister a probe by providing the registered private data.
899 * Only removes the first marker found in hash table.
900 * Return 0 on success or error value.
901 * We do not need to call a synchronize_sched to make sure the probes have
902 * finished running before doing a module unload, because the module unload
903 * itself uses stop_machine(), which insures that every preempt disabled section
906 int marker_probe_unregister_private_data(marker_probe_func
*probe
,
909 struct marker_entry
*entry
;
911 struct marker_probe_closure
*old
;
912 const char *channel
= NULL
, *name
= NULL
;
914 mutex_lock(&markers_mutex
);
915 entry
= get_marker_from_private_data(probe
, probe_private
);
920 if (entry
->rcu_pending
)
922 old
= marker_entry_remove_probe(entry
, NULL
, probe_private
);
923 channel
= kstrdup(entry
->channel
, GFP_KERNEL
);
924 name
= kstrdup(entry
->name
, GFP_KERNEL
);
925 mutex_unlock(&markers_mutex
);
927 marker_update_probes();
929 mutex_lock(&markers_mutex
);
930 entry
= get_marker(channel
, name
);
933 if (entry
->rcu_pending
)
936 entry
->rcu_pending
= 1;
937 /* write rcu_pending before calling the RCU callback */
939 call_rcu_sched(&entry
->rcu
, free_old_closure
);
940 /* Ignore busy error message */
941 remove_marker(channel
, name
);
943 mutex_unlock(&markers_mutex
);
948 EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data
);
951 * marker_get_private_data - Get a marker's probe private data
952 * @channel: marker channel
954 * @probe: probe to match
955 * @num: get the nth matching probe's private data
957 * Returns the nth private data pointer (starting from 0) matching, or an
959 * Returns the private data pointer, or an ERR_PTR.
960 * The private data pointer should _only_ be dereferenced if the caller is the
961 * owner of the data, or its content could vanish. This is mostly used to
962 * confirm that a caller is the owner of a registered probe.
964 void *marker_get_private_data(const char *channel
, const char *name
,
965 marker_probe_func
*probe
, int num
)
967 struct hlist_head
*head
;
968 struct hlist_node
*node
;
969 struct marker_entry
*e
;
970 size_t channel_len
= strlen(channel
) + 1;
971 size_t name_len
= strlen(name
) + 1;
975 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
976 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
977 hlist_for_each_entry(e
, node
, head
, hlist
) {
978 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
980 if (num
== 0 && e
->single
.func
== probe
)
981 return e
->single
.probe_private
;
983 struct marker_probe_closure
*closure
;
986 for (i
= 0; closure
[i
].func
; i
++) {
987 if (closure
[i
].func
!= probe
)
990 return closure
[i
].probe_private
;
996 return ERR_PTR(-ENOENT
);
998 EXPORT_SYMBOL_GPL(marker_get_private_data
);
1001 * markers_compact_event_ids - Compact markers event IDs and reassign channels
1003 * Called when no channel users are active by the channel infrastructure.
1004 * Called with lock_markers() and channel mutex held.
1006 void markers_compact_event_ids(void)
1008 struct marker_entry
*entry
;
1010 struct hlist_head
*head
;
1011 struct hlist_node
*node
;
1014 for (i
= 0; i
< MARKER_TABLE_SIZE
; i
++) {
1015 head
= &marker_table
[i
];
1016 hlist_for_each_entry(entry
, node
, head
, hlist
) {
1017 ret
= ltt_channels_get_index_from_name(entry
->channel
);
1019 entry
->channel_id
= ret
;
1020 ret
= _ltt_channels_get_event_id(entry
->channel
,
1023 entry
->event_id
= ret
;
1028 #ifdef CONFIG_MODULES
1031 * marker_get_iter_range - Get a next marker iterator given a range.
1032 * @marker: current markers (in), next marker (out)
1033 * @begin: beginning of the range
1034 * @end: end of the range
1036 * Returns whether a next marker has been found (1) or not (0).
1037 * Will return the first marker in the range if the input marker is NULL.
1039 int marker_get_iter_range(struct marker
**marker
, struct marker
*begin
,
1042 if (!*marker
&& begin
!= end
) {
1046 if (*marker
>= begin
&& *marker
< end
)
1050 EXPORT_SYMBOL_GPL(marker_get_iter_range
);
1052 static void marker_get_iter(struct marker_iter
*iter
)
1056 /* Core kernel markers */
1057 if (!iter
->module
) {
1058 found
= marker_get_iter_range(&iter
->marker
,
1059 __start___markers
, __stop___markers
);
1063 /* Markers in modules. */
1064 found
= module_get_iter_markers(iter
);
1067 marker_iter_reset(iter
);
1070 void marker_iter_start(struct marker_iter
*iter
)
1072 marker_get_iter(iter
);
1074 EXPORT_SYMBOL_GPL(marker_iter_start
);
1076 void marker_iter_next(struct marker_iter
*iter
)
1080 * iter->marker may be invalid because we blindly incremented it.
1081 * Make sure it is valid by marshalling on the markers, getting the
1082 * markers from following modules if necessary.
1084 marker_get_iter(iter
);
1086 EXPORT_SYMBOL_GPL(marker_iter_next
);
1088 void marker_iter_stop(struct marker_iter
*iter
)
1091 EXPORT_SYMBOL_GPL(marker_iter_stop
);
1093 void marker_iter_reset(struct marker_iter
*iter
)
1095 iter
->module
= NULL
;
1096 iter
->marker
= NULL
;
1098 EXPORT_SYMBOL_GPL(marker_iter_reset
);
1100 #ifdef CONFIG_MARKERS_USERSPACE
1102 * must be called with current->user_markers_mutex held
1104 static void free_user_marker(char __user
*state
, struct hlist_head
*head
)
1106 struct user_marker
*umark
;
1107 struct hlist_node
*pos
, *n
;
1109 hlist_for_each_entry_safe(umark
, pos
, n
, head
, hlist
) {
1110 if (umark
->state
== state
) {
1111 hlist_del(&umark
->hlist
);
1117 asmlinkage
long sys_marker(char __user
*name
, char __user
*format
,
1118 char __user
*state
, int reg
)
1120 struct user_marker
*umark
;
1122 struct marker_entry
*entry
;
1125 printk(KERN_DEBUG
"Program %s %s marker [%p, %p]\n",
1126 current
->comm
, reg
? "registers" : "unregisters",
1129 umark
= kmalloc(sizeof(struct user_marker
), GFP_KERNEL
);
1130 umark
->name
[MAX_USER_MARKER_NAME_LEN
- 1] = '\0';
1131 umark
->format
[MAX_USER_MARKER_FORMAT_LEN
- 1] = '\0';
1132 umark
->state
= state
;
1133 len
= strncpy_from_user(umark
->name
, name
,
1134 MAX_USER_MARKER_NAME_LEN
- 1);
1139 len
= strncpy_from_user(umark
->format
, format
,
1140 MAX_USER_MARKER_FORMAT_LEN
- 1);
1145 printk(KERN_DEBUG
"Marker name : %s, format : %s", umark
->name
,
1147 mutex_lock(&markers_mutex
);
1148 entry
= get_marker("userspace", umark
->name
);
1150 if (entry
->format
&&
1151 strcmp(entry
->format
, umark
->format
) != 0) {
1152 printk(" error, wrong format in process %s",
1157 printk(" %s", !!entry
->refcount
1158 ? "enabled" : "disabled");
1159 if (put_user(!!entry
->refcount
, state
)) {
1165 printk(" disabled\n");
1166 if (put_user(0, umark
->state
)) {
1168 "Marker in %s caused a fault\n",
1173 mutex_lock(¤t
->group_leader
->user_markers_mutex
);
1174 hlist_add_head(&umark
->hlist
,
1175 ¤t
->group_leader
->user_markers
);
1176 current
->group_leader
->user_markers_sequence
++;
1177 mutex_unlock(¤t
->group_leader
->user_markers_mutex
);
1178 mutex_unlock(&markers_mutex
);
1180 mutex_lock(¤t
->group_leader
->user_markers_mutex
);
1181 free_user_marker(state
,
1182 ¤t
->group_leader
->user_markers
);
1183 current
->group_leader
->user_markers_sequence
++;
1184 mutex_unlock(¤t
->group_leader
->user_markers_mutex
);
1188 mutex_unlock(&markers_mutex
);
1199 asmlinkage
long sys_trace(int type
, uint16_t id
,
1207 case 0: /* String */
1209 page
= (char *)__get_free_page(GFP_TEMPORARY
);
1212 len
= strncpy_from_user(page
, ubuf
, PAGE_SIZE
);
1217 trace_mark(userspace
, string
, "string %s", page
);
1219 free_page((unsigned long) page
);
1228 static void marker_update_processes(void)
1230 struct task_struct
*g
, *t
;
1233 * markers_mutex is taken to protect the p->user_markers read.
1235 mutex_lock(&markers_mutex
);
1236 read_lock(&tasklist_lock
);
1237 for_each_process(g
) {
1238 WARN_ON(!thread_group_leader(g
));
1239 if (hlist_empty(&g
->user_markers
))
1241 if (strcmp(g
->comm
, "testprog") == 0)
1242 printk(KERN_DEBUG
"set update pending for testprog\n");
1245 /* TODO : implement this thread flag in each arch. */
1246 set_tsk_thread_flag(t
, TIF_MARKER_PENDING
);
1247 } while ((t
= next_thread(t
)) != g
);
1249 read_unlock(&tasklist_lock
);
1250 mutex_unlock(&markers_mutex
);
1254 * Update current process.
1255 * Note that we have to wait a whole scheduler period before we are sure that
1256 * every running userspace threads have their markers updated.
1257 * (synchronize_sched() can be used to insure this).
1259 void marker_update_process(void)
1261 struct user_marker
*umark
;
1262 struct hlist_node
*pos
;
1263 struct marker_entry
*entry
;
1265 mutex_lock(&markers_mutex
);
1266 mutex_lock(¤t
->group_leader
->user_markers_mutex
);
1267 if (strcmp(current
->comm
, "testprog") == 0)
1268 printk(KERN_DEBUG
"do update pending for testprog\n");
1269 hlist_for_each_entry(umark
, pos
,
1270 ¤t
->group_leader
->user_markers
, hlist
) {
1271 printk(KERN_DEBUG
"Updating marker %s in %s\n",
1272 umark
->name
, current
->comm
);
1273 entry
= get_marker("userspace", umark
->name
);
1275 if (entry
->format
&&
1276 strcmp(entry
->format
, umark
->format
) != 0) {
1278 " error, wrong format in process %s\n",
1282 if (put_user(!!entry
->refcount
, umark
->state
)) {
1284 "Marker in %s caused a fault\n",
1289 if (put_user(0, umark
->state
)) {
1291 "Marker in %s caused a fault\n",
1297 clear_thread_flag(TIF_MARKER_PENDING
);
1298 mutex_unlock(¤t
->group_leader
->user_markers_mutex
);
1299 mutex_unlock(&markers_mutex
);
1303 * Called at process exit and upon do_execve().
1304 * We assume that when the leader exits, no more references can be done to the
1305 * leader structure by the other threads.
1307 void exit_user_markers(struct task_struct
*p
)
1309 struct user_marker
*umark
;
1310 struct hlist_node
*pos
, *n
;
1312 if (thread_group_leader(p
)) {
1313 mutex_lock(&markers_mutex
);
1314 mutex_lock(&p
->user_markers_mutex
);
1315 hlist_for_each_entry_safe(umark
, pos
, n
, &p
->user_markers
,
1318 INIT_HLIST_HEAD(&p
->user_markers
);
1319 p
->user_markers_sequence
++;
1320 mutex_unlock(&p
->user_markers_mutex
);
1321 mutex_unlock(&markers_mutex
);
1325 int is_marker_enabled(const char *channel
, const char *name
)
1327 struct marker_entry
*entry
;
1329 mutex_lock(&markers_mutex
);
1330 entry
= get_marker(channel
, name
);
1331 mutex_unlock(&markers_mutex
);
1333 return entry
&& !!entry
->refcount
;
1337 int marker_module_notify(struct notifier_block
*self
,
1338 unsigned long val
, void *data
)
1340 struct module
*mod
= data
;
1343 case MODULE_STATE_COMING
:
1344 marker_update_probe_range(mod
->markers
,
1345 mod
->markers
+ mod
->num_markers
);
1347 case MODULE_STATE_GOING
:
1348 marker_update_probe_range(mod
->markers
,
1349 mod
->markers
+ mod
->num_markers
);
1355 struct notifier_block marker_module_nb
= {
1356 .notifier_call
= marker_module_notify
,
1360 static int init_markers(void)
1362 return register_module_notifier(&marker_module_nb
);
1364 __initcall(init_markers
);
1366 #endif /* CONFIG_MODULES */
1368 void ltt_dump_marker_state(struct ltt_trace_struct
*trace
)
1370 struct marker_iter iter
;
1371 struct ltt_probe_private_data call_data
;
1372 const char *channel
;
1374 call_data
.trace
= trace
;
1375 call_data
.serializer
= NULL
;
1377 marker_iter_reset(&iter
);
1378 marker_iter_start(&iter
);
1379 for (; iter
.marker
!= NULL
; marker_iter_next(&iter
)) {
1380 if (!_imv_read(iter
.marker
->state
))
1382 channel
= ltt_channels_get_name_from_index(
1383 iter
.marker
->channel_id
);
1384 __trace_mark(0, metadata
, core_marker_id
,
1386 "channel %s name %s event_id %hu "
1387 "int #1u%zu long #1u%zu pointer #1u%zu "
1388 "size_t #1u%zu alignment #1u%u",
1391 iter
.marker
->event_id
,
1392 sizeof(int), sizeof(long),
1393 sizeof(void *), sizeof(size_t),
1394 ltt_get_alignment());
1395 if (iter
.marker
->format
)
1396 __trace_mark(0, metadata
,
1399 "channel %s name %s format %s",
1402 iter
.marker
->format
);
1404 marker_iter_stop(&iter
);
1406 EXPORT_SYMBOL_GPL(ltt_dump_marker_state
);