2 * Copyright (C) 2007 Mathieu Desnoyers
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 //ust// #include <linux/module.h>
19 //ust// #include <linux/mutex.h>
20 //ust// #include <linux/types.h>
24 //ust// #include <linux/marker.h>
26 //ust// #include <linux/slab.h>
27 //ust// #include <linux/immediate.h>
28 //ust// #include <linux/sched.h>
29 //ust// #include <linux/uaccess.h>
30 //ust// #include <linux/user_marker.h>
31 //ust// #include <linux/ltt-tracer.h>
34 #include "kernelcompat.h"
37 #include "tracercore.h"
40 extern struct marker __start___markers
[] __attribute__((visibility("hidden")));
41 extern struct marker __stop___markers
[] __attribute__((visibility("hidden")));
43 /* Set to 1 to enable marker debug output */
44 static const int marker_debug
;
47 * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
48 * and module markers and the hash table.
50 static DEFINE_MUTEX(markers_mutex
);
52 void lock_markers(void)
54 mutex_lock(&markers_mutex
);
57 void unlock_markers(void)
59 mutex_unlock(&markers_mutex
);
63 * Marker hash table, containing the active markers.
64 * Protected by module_mutex.
66 #define MARKER_HASH_BITS 6
67 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
68 static struct hlist_head marker_table
[MARKER_TABLE_SIZE
];
72 * It is used to make sure every handler has finished using its private data
73 * between two consecutive operation (add or remove) on a given marker. It is
74 * also used to delay the free of multiple probes array until a quiescent state
76 * marker entries modifications are protected by the markers_mutex.
79 struct hlist_node hlist
;
83 void (*call
)(const struct marker
*mdata
, void *call_private
, ...);
84 struct marker_probe_closure single
;
85 struct marker_probe_closure
*multi
;
86 int refcount
; /* Number of times armed. 0 if disarmed. */
92 unsigned char ptype
:1;
93 unsigned char format_allocated
:1;
94 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
97 #ifdef CONFIG_MARKERS_USERSPACE
98 static void marker_update_processes(void);
100 static void marker_update_processes(void)
106 * __mark_empty_function - Empty probe callback
107 * @mdata: marker data
108 * @probe_private: probe private data
109 * @call_private: call site private data
110 * @fmt: format string
111 * @...: variable argument list
113 * Empty callback provided as a probe to the markers. By providing this to a
114 * disabled marker, we make sure the execution flow is always valid even
115 * though the function pointer change and the marker enabling are two distinct
116 * operations that modifies the execution flow of preemptible code.
118 notrace
void __mark_empty_function(const struct marker
*mdata
,
119 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
122 //ust// EXPORT_SYMBOL_GPL(__mark_empty_function);
125 * marker_probe_cb Callback that prepares the variable argument list for probes.
126 * @mdata: pointer of type struct marker
127 * @call_private: caller site private data
128 * @...: Variable argument list.
130 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
131 * need to put a full smp_rmb() in this branch. This is why we do not use
132 * rcu_dereference() for the pointer read.
134 notrace
void marker_probe_cb(const struct marker
*mdata
,
135 void *call_private
, ...)
141 * rcu_read_lock_sched does two things : disabling preemption to make
142 * sure the teardown of the callbacks can be done correctly when they
143 * are in modules and they insure RCU read coherency.
145 //ust// rcu_read_lock_sched_notrace();
146 ptype
= mdata
->ptype
;
147 if (likely(!ptype
)) {
148 marker_probe_func
*func
;
149 /* Must read the ptype before ptr. They are not data dependant,
150 * so we put an explicit smp_rmb() here. */
152 func
= mdata
->single
.func
;
153 /* Must read the ptr before private data. They are not data
154 * dependant, so we put an explicit smp_rmb() here. */
156 va_start(args
, call_private
);
157 func(mdata
, mdata
->single
.probe_private
, call_private
,
158 mdata
->format
, &args
);
161 struct marker_probe_closure
*multi
;
164 * Read mdata->ptype before mdata->multi.
167 multi
= mdata
->multi
;
169 * multi points to an array, therefore accessing the array
170 * depends on reading multi. However, even in this case,
171 * we must insure that the pointer is read _before_ the array
172 * data. Same as rcu_dereference, but we need a full smp_rmb()
173 * in the fast path, so put the explicit barrier here.
175 smp_read_barrier_depends();
176 for (i
= 0; multi
[i
].func
; i
++) {
177 va_start(args
, call_private
);
178 multi
[i
].func(mdata
, multi
[i
].probe_private
,
179 call_private
, mdata
->format
, &args
);
183 //ust// rcu_read_unlock_sched_notrace();
185 //ust// EXPORT_SYMBOL_GPL(marker_probe_cb);
188 * marker_probe_cb Callback that does not prepare the variable argument list.
189 * @mdata: pointer of type struct marker
190 * @call_private: caller site private data
191 * @...: Variable argument list.
193 * Should be connected to markers "MARK_NOARGS".
195 static notrace
void marker_probe_cb_noarg(const struct marker
*mdata
,
196 void *call_private
, ...)
198 va_list args
; /* not initialized */
201 //ust// rcu_read_lock_sched_notrace();
202 ptype
= mdata
->ptype
;
203 if (likely(!ptype
)) {
204 marker_probe_func
*func
;
205 /* Must read the ptype before ptr. They are not data dependant,
206 * so we put an explicit smp_rmb() here. */
208 func
= mdata
->single
.func
;
209 /* Must read the ptr before private data. They are not data
210 * dependant, so we put an explicit smp_rmb() here. */
212 func(mdata
, mdata
->single
.probe_private
, call_private
,
213 mdata
->format
, &args
);
215 struct marker_probe_closure
*multi
;
218 * Read mdata->ptype before mdata->multi.
221 multi
= mdata
->multi
;
223 * multi points to an array, therefore accessing the array
224 * depends on reading multi. However, even in this case,
225 * we must insure that the pointer is read _before_ the array
226 * data. Same as rcu_dereference, but we need a full smp_rmb()
227 * in the fast path, so put the explicit barrier here.
229 smp_read_barrier_depends();
230 for (i
= 0; multi
[i
].func
; i
++)
231 multi
[i
].func(mdata
, multi
[i
].probe_private
,
232 call_private
, mdata
->format
, &args
);
234 //ust// rcu_read_unlock_sched_notrace();
237 static void free_old_closure(struct rcu_head
*head
)
239 struct marker_entry
*entry
= container_of(head
,
240 struct marker_entry
, rcu
);
241 kfree(entry
->oldptr
);
242 /* Make sure we free the data before setting the pending flag to 0 */
244 entry
->rcu_pending
= 0;
247 static void debug_print_probes(struct marker_entry
*entry
)
255 printk(KERN_DEBUG
"Single probe : %p %p\n",
257 entry
->single
.probe_private
);
259 for (i
= 0; entry
->multi
[i
].func
; i
++)
260 printk(KERN_DEBUG
"Multi probe %d : %p %p\n", i
,
261 entry
->multi
[i
].func
,
262 entry
->multi
[i
].probe_private
);
266 static struct marker_probe_closure
*
267 marker_entry_add_probe(struct marker_entry
*entry
,
268 marker_probe_func
*probe
, void *probe_private
)
271 struct marker_probe_closure
*old
, *new;
275 debug_print_probes(entry
);
278 if (entry
->single
.func
== probe
&&
279 entry
->single
.probe_private
== probe_private
)
280 return ERR_PTR(-EBUSY
);
281 if (entry
->single
.func
== __mark_empty_function
) {
283 entry
->single
.func
= probe
;
284 entry
->single
.probe_private
= probe_private
;
287 debug_print_probes(entry
);
295 /* (N -> N+1), (N != 0, 1) probes */
296 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++)
297 if (old
[nr_probes
].func
== probe
298 && old
[nr_probes
].probe_private
300 return ERR_PTR(-EBUSY
);
302 /* + 2 : one for new probe, one for NULL func */
303 new = kzalloc((nr_probes
+ 2) * sizeof(struct marker_probe_closure
),
306 return ERR_PTR(-ENOMEM
);
308 new[0] = entry
->single
;
311 nr_probes
* sizeof(struct marker_probe_closure
));
312 new[nr_probes
].func
= probe
;
313 new[nr_probes
].probe_private
= probe_private
;
314 entry
->refcount
= nr_probes
+ 1;
317 debug_print_probes(entry
);
321 static struct marker_probe_closure
*
322 marker_entry_remove_probe(struct marker_entry
*entry
,
323 marker_probe_func
*probe
, void *probe_private
)
325 int nr_probes
= 0, nr_del
= 0, i
;
326 struct marker_probe_closure
*old
, *new;
330 debug_print_probes(entry
);
332 /* 0 -> N is an error */
333 WARN_ON(entry
->single
.func
== __mark_empty_function
);
335 WARN_ON(probe
&& entry
->single
.func
!= probe
);
336 WARN_ON(entry
->single
.probe_private
!= probe_private
);
337 entry
->single
.func
= __mark_empty_function
;
340 debug_print_probes(entry
);
343 /* (N -> M), (N > 1, M >= 0) probes */
344 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
345 if ((!probe
|| old
[nr_probes
].func
== probe
)
346 && old
[nr_probes
].probe_private
352 if (nr_probes
- nr_del
== 0) {
353 /* N -> 0, (N > 1) */
354 entry
->single
.func
= __mark_empty_function
;
357 } else if (nr_probes
- nr_del
== 1) {
358 /* N -> 1, (N > 1) */
359 for (i
= 0; old
[i
].func
; i
++)
360 if ((probe
&& old
[i
].func
!= probe
) ||
361 old
[i
].probe_private
!= probe_private
)
362 entry
->single
= old
[i
];
367 /* N -> M, (N > 1, M > 1) */
369 new = kzalloc((nr_probes
- nr_del
+ 1)
370 * sizeof(struct marker_probe_closure
), GFP_KERNEL
);
372 return ERR_PTR(-ENOMEM
);
373 for (i
= 0; old
[i
].func
; i
++)
374 if ((probe
&& old
[i
].func
!= probe
) ||
375 old
[i
].probe_private
!= probe_private
)
377 entry
->refcount
= nr_probes
- nr_del
;
381 debug_print_probes(entry
);
386 * Get marker if the marker is present in the marker hash table.
387 * Must be called with markers_mutex held.
388 * Returns NULL if not present.
390 static struct marker_entry
*get_marker(const char *channel
, const char *name
)
392 struct hlist_head
*head
;
393 struct hlist_node
*node
;
394 struct marker_entry
*e
;
395 size_t channel_len
= strlen(channel
) + 1;
396 size_t name_len
= strlen(name
) + 1;
399 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
400 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
401 hlist_for_each_entry(e
, node
, head
, hlist
) {
402 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
409 * Add the marker to the marker hash table. Must be called with markers_mutex
412 static struct marker_entry
*add_marker(const char *channel
, const char *name
,
415 struct hlist_head
*head
;
416 struct hlist_node
*node
;
417 struct marker_entry
*e
;
418 size_t channel_len
= strlen(channel
) + 1;
419 size_t name_len
= strlen(name
) + 1;
420 size_t format_len
= 0;
423 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
425 format_len
= strlen(format
) + 1;
426 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
427 hlist_for_each_entry(e
, node
, head
, hlist
) {
428 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
430 "Marker %s.%s busy\n", channel
, name
);
431 return ERR_PTR(-EBUSY
); /* Already there */
435 * Using kmalloc here to allocate a variable length element. Could
436 * cause some memory fragmentation if overused.
438 e
= kmalloc(sizeof(struct marker_entry
)
439 + channel_len
+ name_len
+ format_len
,
442 return ERR_PTR(-ENOMEM
);
443 memcpy(e
->channel
, channel
, channel_len
);
444 e
->name
= &e
->channel
[channel_len
];
445 memcpy(e
->name
, name
, name_len
);
447 e
->format
= &e
->name
[channel_len
+ name_len
];
448 memcpy(e
->format
, format
, format_len
);
449 if (strcmp(e
->format
, MARK_NOARGS
) == 0)
450 e
->call
= marker_probe_cb_noarg
;
452 e
->call
= marker_probe_cb
;
453 trace_mark(metadata
, core_marker_format
,
454 "channel %s name %s format %s",
455 e
->channel
, e
->name
, e
->format
);
458 e
->call
= marker_probe_cb
;
460 e
->single
.func
= __mark_empty_function
;
461 e
->single
.probe_private
= NULL
;
464 e
->format_allocated
= 0;
467 hlist_add_head(&e
->hlist
, head
);
472 * Remove the marker from the marker hash table. Must be called with mutex_lock
475 static int remove_marker(const char *channel
, const char *name
)
477 struct hlist_head
*head
;
478 struct hlist_node
*node
;
479 struct marker_entry
*e
;
481 size_t channel_len
= strlen(channel
) + 1;
482 size_t name_len
= strlen(name
) + 1;
486 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
487 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
488 hlist_for_each_entry(e
, node
, head
, hlist
) {
489 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
496 if (e
->single
.func
!= __mark_empty_function
)
498 hlist_del(&e
->hlist
);
499 if (e
->format_allocated
)
501 ret
= ltt_channels_unregister(e
->channel
);
503 /* Make sure the call_rcu has been executed */
511 * Set the mark_entry format to the format found in the element.
513 static int marker_set_format(struct marker_entry
*entry
, const char *format
)
515 entry
->format
= kstrdup(format
, GFP_KERNEL
);
518 entry
->format_allocated
= 1;
520 trace_mark(metadata
, core_marker_format
,
521 "channel %s name %s format %s",
522 entry
->channel
, entry
->name
, entry
->format
);
527 * Sets the probe callback corresponding to one marker.
529 static int set_marker(struct marker_entry
*entry
, struct marker
*elem
,
533 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
536 if (strcmp(entry
->format
, elem
->format
) != 0) {
538 "Format mismatch for probe %s "
539 "(%s), marker (%s)\n",
546 ret
= marker_set_format(entry
, elem
->format
);
552 * probe_cb setup (statically known) is done here. It is
553 * asynchronous with the rest of execution, therefore we only
554 * pass from a "safe" callback (with argument) to an "unsafe"
555 * callback (does not set arguments).
557 elem
->call
= entry
->call
;
558 elem
->channel_id
= entry
->channel_id
;
559 elem
->event_id
= entry
->event_id
;
562 * We only update the single probe private data when the ptr is
563 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
565 WARN_ON(elem
->single
.func
!= __mark_empty_function
566 && elem
->single
.probe_private
!= entry
->single
.probe_private
568 elem
->single
.probe_private
= entry
->single
.probe_private
;
570 * Make sure the private data is valid when we update the
574 elem
->single
.func
= entry
->single
.func
;
576 * We also make sure that the new probe callbacks array is consistent
577 * before setting a pointer to it.
579 rcu_assign_pointer(elem
->multi
, entry
->multi
);
581 * Update the function or multi probe array pointer before setting the
585 elem
->ptype
= entry
->ptype
;
587 //ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) {
588 //ust// WARN_ON(!elem->tp_cb);
590 //ust// * It is ok to directly call the probe registration because type
591 //ust// * checking has been done in the __trace_mark_tp() macro.
594 //ust// if (active) {
596 //ust// * try_module_get should always succeed because we hold
597 //ust// * markers_mutex to get the tp_cb address.
599 //ust// ret = try_module_get(__module_text_address(
600 //ust// (unsigned long)elem->tp_cb));
601 //ust// BUG_ON(!ret);
602 //ust// ret = tracepoint_probe_register_noupdate(
603 //ust// elem->tp_name,
604 //ust// elem->tp_cb);
606 //ust// ret = tracepoint_probe_unregister_noupdate(
607 //ust// elem->tp_name,
608 //ust// elem->tp_cb);
610 //ust// * tracepoint_probe_update_all() must be called
611 //ust// * before the module containing tp_cb is unloaded.
613 //ust// module_put(__module_text_address(
614 //ust// (unsigned long)elem->tp_cb));
617 elem
->state__imv
= active
;
623 * Disable a marker and its probe callback.
624 * Note: only waiting an RCU period after setting elem->call to the empty
625 * function insures that the original callback is not used anymore. This insured
626 * by rcu_read_lock_sched around the call site.
628 static void disable_marker(struct marker
*elem
)
632 /* leave "call" as is. It is known statically. */
633 //ust// if (elem->tp_name && _imv_read(elem->state)) {
634 //ust// WARN_ON(!elem->tp_cb);
636 //ust// * It is ok to directly call the probe registration because type
637 //ust// * checking has been done in the __trace_mark_tp() macro.
639 //ust// ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
640 //ust// elem->tp_cb);
641 //ust// WARN_ON(ret);
643 //ust// * tracepoint_probe_update_all() must be called
644 //ust// * before the module containing tp_cb is unloaded.
646 //ust// module_put(__module_text_address((unsigned long)elem->tp_cb));
648 elem
->state__imv
= 0;
649 elem
->single
.func
= __mark_empty_function
;
650 /* Update the function before setting the ptype */
652 elem
->ptype
= 0; /* single probe */
654 * Leave the private data and channel_id/event_id there, because removal
655 * is racy and should be done only after an RCU period. These are never
656 * used until the next initialization anyway.
661 * marker_update_probe_range - Update a probe range
662 * @begin: beginning of the range
663 * @end: end of the range
665 * Updates the probe callback corresponding to a range of markers.
667 void marker_update_probe_range(struct marker
*begin
,
671 struct marker_entry
*mark_entry
;
673 mutex_lock(&markers_mutex
);
674 for (iter
= begin
; iter
< end
; iter
++) {
675 mark_entry
= get_marker(iter
->channel
, iter
->name
);
677 set_marker(mark_entry
, iter
, !!mark_entry
->refcount
);
679 * ignore error, continue
682 /* This is added for UST. We emit a core_marker_id event
683 * for markers that are already registered to a probe
684 * upon library load. Otherwise, no core_marker_id will
685 * be generated for these markers. Is this the right thing
688 trace_mark(metadata
, core_marker_id
,
689 "channel %s name %s event_id %hu "
690 "int #1u%zu long #1u%zu pointer #1u%zu "
691 "size_t #1u%zu alignment #1u%u",
692 iter
->channel
, iter
->name
, mark_entry
->event_id
,
693 sizeof(int), sizeof(long), sizeof(void *),
694 sizeof(size_t), ltt_get_alignment());
696 disable_marker(iter
);
699 mutex_unlock(&markers_mutex
);
703 * Update probes, removing the faulty probes.
705 * Internal callback only changed before the first probe is connected to it.
706 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
707 * transitions. All other transitions will leave the old private data valid.
708 * This makes the non-atomicity of the callback/private data updates valid.
710 * "special case" updates :
715 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
716 * Site effect : marker_set_format may delete the marker entry (creating a
719 static void marker_update_probes(void)
721 /* Core kernel markers */
722 //ust// marker_update_probe_range(__start___markers, __stop___markers);
723 /* Markers in modules. */
724 //ust// module_update_markers();
725 lib_update_markers();
726 //ust// tracepoint_probe_update_all();
727 /* Update immediate values */
729 //ust// module_imv_update();
730 marker_update_processes();
734 * marker_probe_register - Connect a probe to a marker
735 * @channel: marker channel
737 * @format: format string
738 * @probe: probe handler
739 * @probe_private: probe private data
741 * private data must be a valid allocated memory address, or NULL.
742 * Returns 0 if ok, error value on error.
743 * The probe address must at least be aligned on the architecture pointer size.
745 int marker_probe_register(const char *channel
, const char *name
,
746 const char *format
, marker_probe_func
*probe
,
749 struct marker_entry
*entry
;
750 int ret
= 0, ret_err
;
751 struct marker_probe_closure
*old
;
754 mutex_lock(&markers_mutex
);
755 entry
= get_marker(channel
, name
);
758 entry
= add_marker(channel
, name
, format
);
760 ret
= PTR_ERR(entry
);
763 ret
= ltt_channels_register(channel
);
765 goto error_remove_marker
;
766 ret
= ltt_channels_get_index_from_name(channel
);
768 goto error_unregister_channel
;
769 entry
->channel_id
= ret
;
770 ret
= ltt_channels_get_event_id(channel
, name
);
772 goto error_unregister_channel
;
773 entry
->event_id
= ret
;
775 trace_mark(metadata
, core_marker_id
,
776 "channel %s name %s event_id %hu "
777 "int #1u%zu long #1u%zu pointer #1u%zu "
778 "size_t #1u%zu alignment #1u%u",
779 channel
, name
, entry
->event_id
,
780 sizeof(int), sizeof(long), sizeof(void *),
781 sizeof(size_t), ltt_get_alignment());
784 ret
= marker_set_format(entry
, format
);
785 else if (strcmp(entry
->format
, format
))
792 * If we detect that a call_rcu is pending for this marker,
793 * make sure it's executed now.
795 if (entry
->rcu_pending
)
797 old
= marker_entry_add_probe(entry
, probe
, probe_private
);
801 goto error_unregister_channel
;
805 mutex_unlock(&markers_mutex
);
807 marker_update_probes();
809 mutex_lock(&markers_mutex
);
810 entry
= get_marker(channel
, name
);
813 if (entry
->rcu_pending
)
816 entry
->rcu_pending
= 1;
817 /* write rcu_pending before calling the RCU callback */
819 call_rcu_sched(&entry
->rcu
, free_old_closure
);
822 error_unregister_channel
:
823 ret_err
= ltt_channels_unregister(channel
);
826 ret_err
= remove_marker(channel
, name
);
829 mutex_unlock(&markers_mutex
);
832 //ust// EXPORT_SYMBOL_GPL(marker_probe_register);
835 * marker_probe_unregister - Disconnect a probe from a marker
836 * @channel: marker channel
838 * @probe: probe function pointer
839 * @probe_private: probe private data
841 * Returns the private data given to marker_probe_register, or an ERR_PTR().
842 * We do not need to call a synchronize_sched to make sure the probes have
843 * finished running before doing a module unload, because the module unload
844 * itself uses stop_machine(), which insures that every preempt disabled section
847 int marker_probe_unregister(const char *channel
, const char *name
,
848 marker_probe_func
*probe
, void *probe_private
)
850 struct marker_entry
*entry
;
851 struct marker_probe_closure
*old
;
854 mutex_lock(&markers_mutex
);
855 entry
= get_marker(channel
, name
);
858 if (entry
->rcu_pending
)
860 old
= marker_entry_remove_probe(entry
, probe
, probe_private
);
861 mutex_unlock(&markers_mutex
);
863 marker_update_probes();
865 mutex_lock(&markers_mutex
);
866 entry
= get_marker(channel
, name
);
869 if (entry
->rcu_pending
)
872 entry
->rcu_pending
= 1;
873 /* write rcu_pending before calling the RCU callback */
875 call_rcu_sched(&entry
->rcu
, free_old_closure
);
876 remove_marker(channel
, name
); /* Ignore busy error message */
879 mutex_unlock(&markers_mutex
);
882 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister);
884 static struct marker_entry
*
885 get_marker_from_private_data(marker_probe_func
*probe
, void *probe_private
)
887 struct marker_entry
*entry
;
889 struct hlist_head
*head
;
890 struct hlist_node
*node
;
892 for (i
= 0; i
< MARKER_TABLE_SIZE
; i
++) {
893 head
= &marker_table
[i
];
894 hlist_for_each_entry(entry
, node
, head
, hlist
) {
896 if (entry
->single
.func
== probe
897 && entry
->single
.probe_private
901 struct marker_probe_closure
*closure
;
902 closure
= entry
->multi
;
903 for (i
= 0; closure
[i
].func
; i
++) {
904 if (closure
[i
].func
== probe
&&
905 closure
[i
].probe_private
916 * marker_probe_unregister_private_data - Disconnect a probe from a marker
917 * @probe: probe function
918 * @probe_private: probe private data
920 * Unregister a probe by providing the registered private data.
921 * Only removes the first marker found in hash table.
922 * Return 0 on success or error value.
923 * We do not need to call a synchronize_sched to make sure the probes have
924 * finished running before doing a module unload, because the module unload
925 * itself uses stop_machine(), which insures that every preempt disabled section
928 int marker_probe_unregister_private_data(marker_probe_func
*probe
,
931 struct marker_entry
*entry
;
933 struct marker_probe_closure
*old
;
934 const char *channel
= NULL
, *name
= NULL
;
936 mutex_lock(&markers_mutex
);
937 entry
= get_marker_from_private_data(probe
, probe_private
);
942 if (entry
->rcu_pending
)
944 old
= marker_entry_remove_probe(entry
, NULL
, probe_private
);
945 channel
= kstrdup(entry
->channel
, GFP_KERNEL
);
946 name
= kstrdup(entry
->name
, GFP_KERNEL
);
947 mutex_unlock(&markers_mutex
);
949 marker_update_probes();
951 mutex_lock(&markers_mutex
);
952 entry
= get_marker(channel
, name
);
955 if (entry
->rcu_pending
)
958 entry
->rcu_pending
= 1;
959 /* write rcu_pending before calling the RCU callback */
961 call_rcu_sched(&entry
->rcu
, free_old_closure
);
962 /* Ignore busy error message */
963 remove_marker(channel
, name
);
965 mutex_unlock(&markers_mutex
);
970 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
973 * marker_get_private_data - Get a marker's probe private data
974 * @channel: marker channel
976 * @probe: probe to match
977 * @num: get the nth matching probe's private data
979 * Returns the nth private data pointer (starting from 0) matching, or an
981 * Returns the private data pointer, or an ERR_PTR.
982 * The private data pointer should _only_ be dereferenced if the caller is the
983 * owner of the data, or its content could vanish. This is mostly used to
984 * confirm that a caller is the owner of a registered probe.
986 void *marker_get_private_data(const char *channel
, const char *name
,
987 marker_probe_func
*probe
, int num
)
989 struct hlist_head
*head
;
990 struct hlist_node
*node
;
991 struct marker_entry
*e
;
992 size_t channel_len
= strlen(channel
) + 1;
993 size_t name_len
= strlen(name
) + 1;
997 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
998 head
= &marker_table
[hash
& ((1 << MARKER_HASH_BITS
)-1)];
999 hlist_for_each_entry(e
, node
, head
, hlist
) {
1000 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
1002 if (num
== 0 && e
->single
.func
== probe
)
1003 return e
->single
.probe_private
;
1005 struct marker_probe_closure
*closure
;
1008 for (i
= 0; closure
[i
].func
; i
++) {
1009 if (closure
[i
].func
!= probe
)
1012 return closure
[i
].probe_private
;
1018 return ERR_PTR(-ENOENT
);
1020 //ust// EXPORT_SYMBOL_GPL(marker_get_private_data);
1023 * markers_compact_event_ids - Compact markers event IDs and reassign channels
1025 * Called when no channel users are active by the channel infrastructure.
1026 * Called with lock_markers() and channel mutex held.
1028 //ust// void markers_compact_event_ids(void)
1030 //ust// struct marker_entry *entry;
1031 //ust// unsigned int i;
1032 //ust// struct hlist_head *head;
1033 //ust// struct hlist_node *node;
1036 //ust// for (i = 0; i < MARKER_TABLE_SIZE; i++) {
1037 //ust// head = &marker_table[i];
1038 //ust// hlist_for_each_entry(entry, node, head, hlist) {
1039 //ust// ret = ltt_channels_get_index_from_name(entry->channel);
1040 //ust// WARN_ON(ret < 0);
1041 //ust// entry->channel_id = ret;
1042 //ust// ret = _ltt_channels_get_event_id(entry->channel,
1043 //ust// entry->name);
1044 //ust// WARN_ON(ret < 0);
1045 //ust// entry->event_id = ret;
1050 //ust//#ifdef CONFIG_MODULES
1053 * marker_get_iter_range - Get a next marker iterator given a range.
1054 * @marker: current markers (in), next marker (out)
1055 * @begin: beginning of the range
1056 * @end: end of the range
1058 * Returns whether a next marker has been found (1) or not (0).
1059 * Will return the first marker in the range if the input marker is NULL.
1061 int marker_get_iter_range(struct marker
**marker
, struct marker
*begin
,
1064 if (!*marker
&& begin
!= end
) {
1068 if (*marker
>= begin
&& *marker
< end
)
1072 //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range);
1074 static void marker_get_iter(struct marker_iter
*iter
)
1078 /* Core kernel markers */
1080 /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */
1081 found
= marker_get_iter_range(&iter
->marker
,
1082 __start___markers
, __stop___markers
);
1086 /* Markers in modules. */
1087 found
= lib_get_iter_markers(iter
);
1090 marker_iter_reset(iter
);
1093 void marker_iter_start(struct marker_iter
*iter
)
1095 marker_get_iter(iter
);
1097 //ust// EXPORT_SYMBOL_GPL(marker_iter_start);
1099 void marker_iter_next(struct marker_iter
*iter
)
1103 * iter->marker may be invalid because we blindly incremented it.
1104 * Make sure it is valid by marshalling on the markers, getting the
1105 * markers from following modules if necessary.
1107 marker_get_iter(iter
);
1109 //ust// EXPORT_SYMBOL_GPL(marker_iter_next);
1111 void marker_iter_stop(struct marker_iter
*iter
)
1114 //ust// EXPORT_SYMBOL_GPL(marker_iter_stop);
1116 void marker_iter_reset(struct marker_iter
*iter
)
1119 iter
->marker
= NULL
;
1121 //ust// EXPORT_SYMBOL_GPL(marker_iter_reset);
1123 #ifdef CONFIG_MARKERS_USERSPACE
1125 * must be called with current->user_markers_mutex held
1127 static void free_user_marker(char __user
*state
, struct hlist_head
*head
)
1129 struct user_marker
*umark
;
1130 struct hlist_node
*pos
, *n
;
1132 hlist_for_each_entry_safe(umark
, pos
, n
, head
, hlist
) {
1133 if (umark
->state
== state
) {
1134 hlist_del(&umark
->hlist
);
1140 //ust// asmlinkage long sys_marker(char __user *name, char __user *format,
1141 //ust// char __user *state, int reg)
1143 //ust// struct user_marker *umark;
1145 //ust// struct marker_entry *entry;
1146 //ust// int ret = 0;
1148 //ust// printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n",
1149 //ust// current->comm, reg ? "registers" : "unregisters",
1150 //ust// name, state);
1152 //ust// umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL);
1153 //ust// umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0';
1154 //ust// umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0';
1155 //ust// umark->state = state;
1156 //ust// len = strncpy_from_user(umark->name, name,
1157 //ust// MAX_USER_MARKER_NAME_LEN - 1);
1158 //ust// if (len < 0) {
1159 //ust// ret = -EFAULT;
1162 //ust// len = strncpy_from_user(umark->format, format,
1163 //ust// MAX_USER_MARKER_FORMAT_LEN - 1);
1164 //ust// if (len < 0) {
1165 //ust// ret = -EFAULT;
1168 //ust// printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name,
1169 //ust// umark->format);
1170 //ust// mutex_lock(&markers_mutex);
1171 //ust// entry = get_marker("userspace", umark->name);
1172 //ust// if (entry) {
1173 //ust// if (entry->format &&
1174 //ust// strcmp(entry->format, umark->format) != 0) {
1175 //ust// printk(" error, wrong format in process %s",
1176 //ust// current->comm);
1177 //ust// ret = -EPERM;
1178 //ust// goto error_unlock;
1180 //ust// printk(" %s", !!entry->refcount
1181 //ust// ? "enabled" : "disabled");
1182 //ust// if (put_user(!!entry->refcount, state)) {
1183 //ust// ret = -EFAULT;
1184 //ust// goto error_unlock;
1186 //ust// printk("\n");
1188 //ust// printk(" disabled\n");
1189 //ust// if (put_user(0, umark->state)) {
1190 //ust// printk(KERN_WARNING
1191 //ust// "Marker in %s caused a fault\n",
1192 //ust// current->comm);
1193 //ust// goto error_unlock;
1196 //ust// mutex_lock(¤t->group_leader->user_markers_mutex);
1197 //ust// hlist_add_head(&umark->hlist,
1198 //ust// ¤t->group_leader->user_markers);
1199 //ust// current->group_leader->user_markers_sequence++;
1200 //ust// mutex_unlock(¤t->group_leader->user_markers_mutex);
1201 //ust// mutex_unlock(&markers_mutex);
1203 //ust// mutex_lock(¤t->group_leader->user_markers_mutex);
1204 //ust// free_user_marker(state,
1205 //ust// ¤t->group_leader->user_markers);
1206 //ust// current->group_leader->user_markers_sequence++;
1207 //ust// mutex_unlock(¤t->group_leader->user_markers_mutex);
1210 //ust// error_unlock:
1211 //ust// mutex_unlock(&markers_mutex);
1213 //ust// kfree(umark);
1220 //ust// * string : 0
1222 //ust// asmlinkage long sys_trace(int type, uint16_t id,
1223 //ust// char __user *ubuf)
1225 //ust// long ret = -EPERM;
1229 //ust// switch (type) {
1230 //ust// case 0: /* String */
1231 //ust// ret = -ENOMEM;
1232 //ust// page = (char *)__get_free_page(GFP_TEMPORARY);
1234 //ust// goto string_out;
1235 //ust// len = strncpy_from_user(page, ubuf, PAGE_SIZE);
1236 //ust// if (len < 0) {
1237 //ust// ret = -EFAULT;
1238 //ust// goto string_err;
1240 //ust// trace_mark(userspace, string, "string %s", page);
1242 //ust// free_page((unsigned long) page);
1251 //ust// static void marker_update_processes(void)
1253 //ust// struct task_struct *g, *t;
1256 //ust// * markers_mutex is taken to protect the p->user_markers read.
1258 //ust// mutex_lock(&markers_mutex);
1259 //ust// read_lock(&tasklist_lock);
1260 //ust// for_each_process(g) {
1261 //ust// WARN_ON(!thread_group_leader(g));
1262 //ust// if (hlist_empty(&g->user_markers))
1264 //ust// if (strcmp(g->comm, "testprog") == 0)
1265 //ust// printk(KERN_DEBUG "set update pending for testprog\n");
1268 //ust// /* TODO : implement this thread flag in each arch. */
1269 //ust// set_tsk_thread_flag(t, TIF_MARKER_PENDING);
1270 //ust// } while ((t = next_thread(t)) != g);
1272 //ust// read_unlock(&tasklist_lock);
1273 //ust// mutex_unlock(&markers_mutex);
1277 * Update current process.
1278 * Note that we have to wait a whole scheduler period before we are sure that
1279 * every running userspace threads have their markers updated.
1280 * (synchronize_sched() can be used to insure this).
1282 void marker_update_process(void)
1284 struct user_marker
*umark
;
1285 struct hlist_node
*pos
;
1286 struct marker_entry
*entry
;
1288 mutex_lock(&markers_mutex
);
1289 mutex_lock(¤t
->group_leader
->user_markers_mutex
);
1290 if (strcmp(current
->comm
, "testprog") == 0)
1291 printk(KERN_DEBUG
"do update pending for testprog\n");
1292 hlist_for_each_entry(umark
, pos
,
1293 ¤t
->group_leader
->user_markers
, hlist
) {
1294 printk(KERN_DEBUG
"Updating marker %s in %s\n",
1295 umark
->name
, current
->comm
);
1296 entry
= get_marker("userspace", umark
->name
);
1298 if (entry
->format
&&
1299 strcmp(entry
->format
, umark
->format
) != 0) {
1301 " error, wrong format in process %s\n",
1305 if (put_user(!!entry
->refcount
, umark
->state
)) {
1307 "Marker in %s caused a fault\n",
1312 if (put_user(0, umark
->state
)) {
1314 "Marker in %s caused a fault\n",
1320 clear_thread_flag(TIF_MARKER_PENDING
);
1321 mutex_unlock(¤t
->group_leader
->user_markers_mutex
);
1322 mutex_unlock(&markers_mutex
);
1326 * Called at process exit and upon do_execve().
1327 * We assume that when the leader exits, no more references can be done to the
1328 * leader structure by the other threads.
1330 void exit_user_markers(struct task_struct
*p
)
1332 struct user_marker
*umark
;
1333 struct hlist_node
*pos
, *n
;
1335 if (thread_group_leader(p
)) {
1336 mutex_lock(&markers_mutex
);
1337 mutex_lock(&p
->user_markers_mutex
);
1338 hlist_for_each_entry_safe(umark
, pos
, n
, &p
->user_markers
,
1341 INIT_HLIST_HEAD(&p
->user_markers
);
1342 p
->user_markers_sequence
++;
1343 mutex_unlock(&p
->user_markers_mutex
);
1344 mutex_unlock(&markers_mutex
);
1348 int is_marker_enabled(const char *channel
, const char *name
)
1350 struct marker_entry
*entry
;
1352 mutex_lock(&markers_mutex
);
1353 entry
= get_marker(channel
, name
);
1354 mutex_unlock(&markers_mutex
);
1356 return entry
&& !!entry
->refcount
;
1360 int marker_module_notify(struct notifier_block
*self
,
1361 unsigned long val
, void *data
)
1363 struct module
*mod
= data
;
1366 case MODULE_STATE_COMING
:
1367 marker_update_probe_range(mod
->markers
,
1368 mod
->markers
+ mod
->num_markers
);
1370 case MODULE_STATE_GOING
:
1371 marker_update_probe_range(mod
->markers
,
1372 mod
->markers
+ mod
->num_markers
);
1378 struct notifier_block marker_module_nb
= {
1379 .notifier_call
= marker_module_notify
,
1383 //ust// static int init_markers(void)
1385 //ust// return register_module_notifier(&marker_module_nb);
1387 //ust// __initcall(init_markers);
1388 /* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */
1390 #endif /* CONFIG_MODULES */
1392 void ltt_dump_marker_state(struct ltt_trace_struct
*trace
)
1394 struct marker_iter iter
;
1395 struct ltt_probe_private_data call_data
;
1396 const char *channel
;
1398 call_data
.trace
= trace
;
1399 call_data
.serializer
= NULL
;
1401 marker_iter_reset(&iter
);
1402 marker_iter_start(&iter
);
1403 for (; iter
.marker
!= NULL
; marker_iter_next(&iter
)) {
1404 if (!_imv_read(iter
.marker
->state
))
1406 channel
= ltt_channels_get_name_from_index(
1407 iter
.marker
->channel_id
);
1408 __trace_mark(0, metadata
, core_marker_id
,
1410 "channel %s name %s event_id %hu "
1411 "int #1u%zu long #1u%zu pointer #1u%zu "
1412 "size_t #1u%zu alignment #1u%u",
1415 iter
.marker
->event_id
,
1416 sizeof(int), sizeof(long),
1417 sizeof(void *), sizeof(size_t),
1418 ltt_get_alignment());
1419 if (iter
.marker
->format
)
1420 __trace_mark(0, metadata
,
1423 "channel %s name %s format %s",
1426 iter
.marker
->format
);
1428 marker_iter_stop(&iter
);
1430 //ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state);
1433 static LIST_HEAD(libs
);
1436 * Returns 0 if current not found.
1437 * Returns 1 if current found.
1439 int lib_get_iter_markers(struct marker_iter
*iter
)
1441 struct lib
*iter_lib
;
1444 //ust// mutex_lock(&module_mutex);
1445 list_for_each_entry(iter_lib
, &libs
, list
) {
1446 if (iter_lib
< iter
->lib
)
1448 else if (iter_lib
> iter
->lib
)
1449 iter
->marker
= NULL
;
1450 found
= marker_get_iter_range(&iter
->marker
,
1451 iter_lib
->markers_start
,
1452 iter_lib
->markers_start
+ iter_lib
->markers_count
);
1454 iter
->lib
= iter_lib
;
1458 //ust// mutex_unlock(&module_mutex);
1462 void lib_update_markers(void)
1466 //ust// mutex_lock(&module_mutex);
1467 list_for_each_entry(lib
, &libs
, list
)
1468 marker_update_probe_range(lib
->markers_start
,
1469 lib
->markers_start
+ lib
->markers_count
);
1470 //ust// mutex_unlock(&module_mutex);
1473 int marker_register_lib(struct marker
*markers_start
, int markers_count
)
1477 pl
= (struct lib
*) malloc(sizeof(struct lib
));
1479 pl
->markers_start
= markers_start
;
1480 pl
->markers_count
= markers_count
;
1482 list_add(&pl
->list
, &libs
);
1484 /* FIXME: update just the loaded lib */
1485 lib_update_markers();
1487 printf("just registered a markers section from %p and having %d markers\n", markers_start
, markers_count
);
1492 static int initialized
= 0;
1494 void __attribute__((constructor
)) init_markers(void)
1497 marker_register_lib(__start___markers
, (((long)__stop___markers
)-((long)__start___markers
))/sizeof(struct marker
));
1498 printf("markers_start: %p, markers_stop: %p\n", __start___markers
, __stop___markers
);