2 * Copyright (C) 2007-2011 Mathieu Desnoyers
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation;
7 * version 2.1 of the License.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 #include <urcu/rculist.h>
25 #include <urcu/hlist.h>
28 #include <ust/marker.h>
29 #include <ust/marker-internal.h>
30 #include <ust/tracepoint.h>
31 #include <ust/tracepoint-internal.h>
33 #include "usterr_signal_safe.h"
35 #include "tracercore.h"
38 extern struct ust_marker
* const __start___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
39 extern struct ust_marker
* const __stop___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
41 /* Set to 1 to enable ust_marker debug output */
42 static const int ust_marker_debug
;
43 static int initialized
;
44 static void (*new_ust_marker_cb
)(struct ust_marker
*);
47 * ust_marker mutex protects the builtin and module ust_marker and the
48 * hash table, as well as the ust_marker_libs list.
50 static DEFINE_MUTEX(ust_marker_mutex
);
51 static CDS_LIST_HEAD(ust_marker_libs
);
54 * Allow nested mutex for mutex listing and nested enable.
56 static __thread
int nested_mutex
;
58 void lock_ust_marker(void)
60 if (!(nested_mutex
++))
61 pthread_mutex_lock(&ust_marker_mutex
);
64 void unlock_ust_marker(void)
66 if (!(--nested_mutex
))
67 pthread_mutex_unlock(&ust_marker_mutex
);
71 * ust_marker hash table, containing the active ust_marker.
72 * Protected by ust_marker mutex.
74 #define UST_MARKER_HASH_BITS 6
75 #define UST_MARKER_TABLE_SIZE (1 << UST_MARKER_HASH_BITS)
76 static struct cds_hlist_head ust_marker_table
[UST_MARKER_TABLE_SIZE
];
78 struct ust_marker_probe_array
{
80 struct ust_marker_probe_closure c
[0];
85 * It is used to make sure every handler has finished using its private
86 * data between two consecutive operation (add or remove) on a given
87 * ust_marker. It is also used to delay the free of multiple probes
88 * array until a quiescent state is reached. ust_marker entries
89 * modifications are protected by the ust_marker_mutex.
91 struct ust_marker_entry
{
92 struct cds_hlist_node hlist
;
96 void (*call
)(const struct ust_marker
*mdata
, void *call_private
, ...);
97 struct ust_marker_probe_closure single
;
98 struct ust_marker_probe_array
*multi
;
99 int refcount
; /* Number of times armed. 0 if disarmed. */
102 unsigned char ptype
:1;
103 unsigned char format_allocated
:1;
104 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
108 * __ust_marker_empty_function - Empty probe callback
109 * @mdata: ust_marker data
110 * @probe_private: probe private data
111 * @call_private: call site private data
112 * @fmt: format string
113 * @...: variable argument list
115 * Empty callback provided as a probe to the ust_marker. By providing
116 * this to a disabled ust_marker, we make sure the execution flow is
117 * always valid even though the function pointer change and the
118 * ust_marker enabling are two distinct operations that modifies the
119 * execution flow of preemptible code.
121 notrace
void __ust_marker_empty_function(const struct ust_marker
*mdata
,
122 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
127 * ust_marker_probe_cb Callback that prepares the variable argument list for probes.
128 * @mdata: pointer of type struct ust_marker
129 * @call_private: caller site private data
130 * @...: Variable argument list.
132 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
133 * need to put a full cmm_smp_rmb() in this branch. This is why we do not use
134 * rcu_dereference() for the pointer read.
136 notrace
void ust_marker_probe_cb(const struct ust_marker
*mdata
,
137 void *call_private
, ...)
143 * rcu_read_lock_sched does two things : disabling preemption to make
144 * sure the teardown of the callbacks can be done correctly when they
145 * are in modules and they insure RCU read coherency.
148 ptype
= mdata
->ptype
;
149 if (likely(!ptype
)) {
150 ust_marker_probe_func
*func
;
151 /* Must read the ptype before ptr. They are not data dependant,
152 * so we put an explicit cmm_smp_rmb() here. */
154 func
= mdata
->single
.func
;
155 /* Must read the ptr before private data. They are not data
156 * dependant, so we put an explicit cmm_smp_rmb() here. */
158 va_start(args
, call_private
);
159 func(mdata
, mdata
->single
.probe_private
, call_private
,
160 mdata
->format
, &args
);
163 struct ust_marker_probe_array
*multi
;
166 * Read mdata->ptype before mdata->multi.
169 multi
= mdata
->multi
;
171 * multi points to an array, therefore accessing the array
172 * depends on reading multi. However, even in this case,
173 * we must insure that the pointer is read _before_ the array
174 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
175 * in the fast path, so put the explicit cmm_barrier here.
177 cmm_smp_read_barrier_depends();
178 for (i
= 0; multi
->c
[i
].func
; i
++) {
179 va_start(args
, call_private
);
180 multi
->c
[i
].func(mdata
, multi
->c
[i
].probe_private
,
181 call_private
, mdata
->format
, &args
);
189 * ust_marker_probe_cb Callback that does not prepare the variable argument list.
190 * @mdata: pointer of type struct ust_marker
191 * @call_private: caller site private data
192 * @...: Variable argument list.
194 * Should be connected to ust_marker "UST_MARKER_NOARGS".
196 static notrace
void ust_marker_probe_cb_noarg(const struct ust_marker
*mdata
,
197 void *call_private
, ...)
199 va_list args
; /* not initialized */
203 ptype
= mdata
->ptype
;
204 if (likely(!ptype
)) {
205 ust_marker_probe_func
*func
;
206 /* Must read the ptype before ptr. They are not data dependant,
207 * so we put an explicit cmm_smp_rmb() here. */
209 func
= mdata
->single
.func
;
210 /* Must read the ptr before private data. They are not data
211 * dependant, so we put an explicit cmm_smp_rmb() here. */
213 func(mdata
, mdata
->single
.probe_private
, call_private
,
214 mdata
->format
, &args
);
216 struct ust_marker_probe_array
*multi
;
219 * Read mdata->ptype before mdata->multi.
222 multi
= mdata
->multi
;
224 * multi points to an array, therefore accessing the array
225 * depends on reading multi. However, even in this case,
226 * we must insure that the pointer is read _before_ the array
227 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
228 * in the fast path, so put the explicit cmm_barrier here.
230 cmm_smp_read_barrier_depends();
231 for (i
= 0; multi
->c
[i
].func
; i
++)
232 multi
->c
[i
].func(mdata
, multi
->c
[i
].probe_private
,
233 call_private
, mdata
->format
, &args
);
238 static void free_old_closure(struct rcu_head
*head
)
240 struct ust_marker_probe_array
*multi
=
241 _ust_container_of(head
, struct ust_marker_probe_array
, rcu
);
245 static void debug_print_probes(struct ust_marker_entry
*entry
)
249 if (!ust_marker_debug
)
253 DBG("Single probe : %p %p",
255 entry
->single
.probe_private
);
257 for (i
= 0; entry
->multi
->c
[i
].func
; i
++)
258 DBG("Multi probe %d : %p %p", i
,
259 entry
->multi
->c
[i
].func
,
260 entry
->multi
->c
[i
].probe_private
);
264 static struct ust_marker_probe_array
*
265 ust_marker_entry_add_probe(struct ust_marker_entry
*entry
,
266 ust_marker_probe_func
*probe
, void *probe_private
)
269 struct ust_marker_probe_array
*old
, *new;
273 debug_print_probes(entry
);
276 if (entry
->single
.func
== probe
&&
277 entry
->single
.probe_private
== probe_private
)
278 return ERR_PTR(-EBUSY
);
279 if (entry
->single
.func
== __ust_marker_empty_function
) {
281 entry
->single
.func
= probe
;
282 entry
->single
.probe_private
= probe_private
;
285 debug_print_probes(entry
);
293 /* (N -> N+1), (N != 0, 1) probes */
294 for (nr_probes
= 0; old
->c
[nr_probes
].func
; nr_probes
++)
295 if (old
->c
[nr_probes
].func
== probe
296 && old
->c
[nr_probes
].probe_private
298 return ERR_PTR(-EBUSY
);
300 /* + 2 : one for new probe, one for NULL func */
301 new = zmalloc(sizeof(struct ust_marker_probe_array
)
302 + ((nr_probes
+ 2) * sizeof(struct ust_marker_probe_closure
)));
304 return ERR_PTR(-ENOMEM
);
306 new->c
[0] = entry
->single
;
308 memcpy(&new->c
[0], &old
->c
[0],
309 nr_probes
* sizeof(struct ust_marker_probe_closure
));
310 new->c
[nr_probes
].func
= probe
;
311 new->c
[nr_probes
].probe_private
= probe_private
;
312 entry
->refcount
= nr_probes
+ 1;
315 debug_print_probes(entry
);
319 static struct ust_marker_probe_array
*
320 ust_marker_entry_remove_probe(struct ust_marker_entry
*entry
,
321 ust_marker_probe_func
*probe
, void *probe_private
)
323 int nr_probes
= 0, nr_del
= 0, i
;
324 struct ust_marker_probe_array
*old
, *new;
328 debug_print_probes(entry
);
330 /* 0 -> N is an error */
331 WARN_ON(entry
->single
.func
== __ust_marker_empty_function
);
333 WARN_ON(probe
&& entry
->single
.func
!= probe
);
334 WARN_ON(entry
->single
.probe_private
!= probe_private
);
335 entry
->single
.func
= __ust_marker_empty_function
;
338 debug_print_probes(entry
);
341 /* (N -> M), (N > 1, M >= 0) probes */
342 for (nr_probes
= 0; old
->c
[nr_probes
].func
; nr_probes
++) {
343 if ((!probe
|| old
->c
[nr_probes
].func
== probe
)
344 && old
->c
[nr_probes
].probe_private
350 if (nr_probes
- nr_del
== 0) {
351 /* N -> 0, (N > 1) */
352 entry
->single
.func
= __ust_marker_empty_function
;
355 } else if (nr_probes
- nr_del
== 1) {
356 /* N -> 1, (N > 1) */
357 for (i
= 0; old
->c
[i
].func
; i
++)
358 if ((probe
&& old
->c
[i
].func
!= probe
) ||
359 old
->c
[i
].probe_private
!= probe_private
)
360 entry
->single
= old
->c
[i
];
365 /* N -> M, (N > 1, M > 1) */
367 new = zmalloc(sizeof(struct ust_marker_probe_array
)
368 + ((nr_probes
- nr_del
+ 1) * sizeof(struct ust_marker_probe_closure
)));
370 return ERR_PTR(-ENOMEM
);
371 for (i
= 0; old
->c
[i
].func
; i
++)
372 if ((probe
&& old
->c
[i
].func
!= probe
) ||
373 old
->c
[i
].probe_private
!= probe_private
)
374 new->c
[j
++] = old
->c
[i
];
375 entry
->refcount
= nr_probes
- nr_del
;
379 debug_print_probes(entry
);
384 * Get ust_marker if the ust_marker is present in the ust_marker hash table.
385 * Must be called with ust_marker_mutex held.
386 * Returns NULL if not present.
388 static struct ust_marker_entry
*get_ust_marker(const char *channel
, const char *name
)
390 struct cds_hlist_head
*head
;
391 struct cds_hlist_node
*node
;
392 struct ust_marker_entry
*e
;
393 size_t channel_len
= strlen(channel
) + 1;
394 size_t name_len
= strlen(name
) + 1;
397 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
398 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
399 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
400 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
407 * Add the ust_marker to the ust_marker hash table. Must be called with
408 * ust_marker_mutex held.
410 static struct ust_marker_entry
*add_ust_marker(const char *channel
, const char *name
,
413 struct cds_hlist_head
*head
;
414 struct cds_hlist_node
*node
;
415 struct ust_marker_entry
*e
;
416 size_t channel_len
= strlen(channel
) + 1;
417 size_t name_len
= strlen(name
) + 1;
418 size_t format_len
= 0;
421 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
423 format_len
= strlen(format
) + 1;
424 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
425 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
426 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
427 DBG("ust_marker %s.%s busy", channel
, name
);
428 return ERR_PTR(-EBUSY
); /* Already there */
432 * Using zmalloc here to allocate a variable length element. Could
433 * cause some memory fragmentation if overused.
435 e
= zmalloc(sizeof(struct ust_marker_entry
)
436 + channel_len
+ name_len
+ format_len
);
438 return ERR_PTR(-ENOMEM
);
439 memcpy(e
->channel
, channel
, channel_len
);
440 e
->name
= &e
->channel
[channel_len
];
441 memcpy(e
->name
, name
, name_len
);
443 e
->format
= &e
->name
[name_len
];
444 memcpy(e
->format
, format
, format_len
);
445 if (strcmp(e
->format
, UST_MARKER_NOARGS
) == 0)
446 e
->call
= ust_marker_probe_cb_noarg
;
448 e
->call
= ust_marker_probe_cb
;
449 __ust_marker(metadata
, core_marker_format
, NULL
,
450 "channel %s name %s format %s",
451 e
->channel
, e
->name
, e
->format
);
454 e
->call
= ust_marker_probe_cb
;
456 e
->single
.func
= __ust_marker_empty_function
;
457 e
->single
.probe_private
= NULL
;
460 e
->format_allocated
= 0;
462 cds_hlist_add_head(&e
->hlist
, head
);
467 * Remove the ust_marker from the ust_marker hash table. Must be called with mutex_lock
470 static int remove_ust_marker(const char *channel
, const char *name
)
472 struct cds_hlist_head
*head
;
473 struct cds_hlist_node
*node
;
474 struct ust_marker_entry
*e
;
476 size_t channel_len
= strlen(channel
) + 1;
477 size_t name_len
= strlen(name
) + 1;
481 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
482 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
483 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
484 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
491 if (e
->single
.func
!= __ust_marker_empty_function
)
493 cds_hlist_del(&e
->hlist
);
494 if (e
->format_allocated
)
496 ret
= ltt_channels_unregister(e
->channel
);
503 * Set the mark_entry format to the format found in the element.
505 static int ust_marker_set_format(struct ust_marker_entry
*entry
, const char *format
)
507 entry
->format
= strdup(format
);
510 entry
->format_allocated
= 1;
512 __ust_marker(metadata
, core_marker_format
, NULL
,
513 "channel %s name %s format %s",
514 entry
->channel
, entry
->name
, entry
->format
);
519 * Sets the probe callback corresponding to one ust_marker.
521 static int set_ust_marker(struct ust_marker_entry
*entry
, struct ust_marker
*elem
,
525 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
528 if (strcmp(entry
->format
, elem
->format
) != 0) {
529 ERR("Format mismatch for probe %s (%s), ust_marker (%s)",
536 ret
= ust_marker_set_format(entry
, elem
->format
);
542 * probe_cb setup (statically known) is done here. It is
543 * asynchronous with the rest of execution, therefore we only
544 * pass from a "safe" callback (with argument) to an "unsafe"
545 * callback (does not set arguments).
547 elem
->call
= entry
->call
;
548 elem
->channel_id
= entry
->channel_id
;
549 elem
->event_id
= entry
->event_id
;
552 * We only update the single probe private data when the ptr is
553 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
555 WARN_ON(elem
->single
.func
!= __ust_marker_empty_function
556 && elem
->single
.probe_private
!= entry
->single
.probe_private
558 elem
->single
.probe_private
= entry
->single
.probe_private
;
560 * Make sure the private data is valid when we update the
564 elem
->single
.func
= entry
->single
.func
;
566 * We also make sure that the new probe callbacks array is consistent
567 * before setting a pointer to it.
569 rcu_assign_pointer(elem
->multi
, entry
->multi
);
571 * Update the function or multi probe array pointer before setting the
575 elem
->ptype
= entry
->ptype
;
577 if (elem
->tp_name
&& (active
^ elem
->state
)) {
578 WARN_ON(!elem
->tp_cb
);
580 * It is ok to directly call the probe registration because type
581 * checking has been done in the __ust_marker_tp() macro.
585 ret
= tracepoint_probe_register_noupdate(
590 * tracepoint_probe_update_all() must be called
591 * before the library containing tp_cb is unloaded.
593 ret
= tracepoint_probe_unregister_noupdate(
598 elem
->state
= active
;
604 * Disable a ust_marker and its probe callback.
605 * Note: only waiting an RCU period after setting elem->call to the empty
606 * function insures that the original callback is not used anymore. This insured
607 * by rcu_read_lock around the call site.
609 static void disable_ust_marker(struct ust_marker
*elem
)
613 /* leave "call" as is. It is known statically. */
614 if (elem
->tp_name
&& elem
->state
) {
615 WARN_ON(!elem
->tp_cb
);
617 * It is ok to directly call the probe registration because type
618 * checking has been done in the __ust_marker_tp() macro.
621 * tracepoint_probe_update_all() must be called
622 * before the module containing tp_cb is unloaded.
624 ret
= tracepoint_probe_unregister_noupdate(elem
->tp_name
,
629 elem
->single
.func
= __ust_marker_empty_function
;
630 /* Update the function before setting the ptype */
632 elem
->ptype
= 0; /* single probe */
634 * Leave the private data and channel_id/event_id there, because removal
635 * is racy and should be done only after an RCU period. These are never
636 * used until the next initialization anyway.
641 * is_ust_marker_enabled - Check if a ust_marker is enabled
642 * @channel: channel name
643 * @name: ust_marker name
645 * Returns 1 if the ust_marker is enabled, 0 if disabled.
647 int is_ust_marker_enabled(const char *channel
, const char *name
)
649 struct ust_marker_entry
*entry
;
652 entry
= get_ust_marker(channel
, name
);
655 return entry
&& !!entry
->refcount
;
659 * ust_marker_update_probe_range - Update a probe range
660 * @begin: beginning of the range
661 * @end: end of the range
663 * Updates the probe callback corresponding to a range of ust_marker.
666 void ust_marker_update_probe_range(struct ust_marker
* const *begin
,
667 struct ust_marker
* const *end
)
669 struct ust_marker
* const *iter
;
670 struct ust_marker_entry
*mark_entry
;
672 for (iter
= begin
; iter
< end
; iter
++) {
674 continue; /* skip dummy */
675 mark_entry
= get_ust_marker((*iter
)->channel
, (*iter
)->name
);
677 set_ust_marker(mark_entry
, *iter
, !!mark_entry
->refcount
);
679 * ignore error, continue
682 disable_ust_marker(*iter
);
687 static void lib_update_ust_marker(void)
689 struct ust_marker_lib
*lib
;
692 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
)
693 ust_marker_update_probe_range(lib
->ust_marker_start
,
694 lib
->ust_marker_start
+ lib
->ust_marker_count
);
699 * Update probes, removing the faulty probes.
701 * Internal callback only changed before the first probe is connected to it.
702 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
703 * transitions. All other transitions will leave the old private data valid.
704 * This makes the non-atomicity of the callback/private data updates valid.
706 * "special case" updates :
711 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
712 * Site effect : ust_marker_set_format may delete the ust_marker entry (creating a
715 static void ust_marker_update_probes(void)
717 lib_update_ust_marker();
718 tracepoint_probe_update_all();
722 * ust_marker_probe_register - Connect a probe to a ust_marker
723 * @channel: ust_marker channel
724 * @name: ust_marker name
725 * @format: format string
726 * @probe: probe handler
727 * @probe_private: probe private data
729 * private data must be a valid allocated memory address, or NULL.
730 * Returns 0 if ok, error value on error.
731 * The probe address must at least be aligned on the architecture pointer size.
733 int ust_marker_probe_register(const char *channel
, const char *name
,
734 const char *format
, ust_marker_probe_func
*probe
,
737 struct ust_marker_entry
*entry
;
738 int ret
= 0, ret_err
;
739 struct ust_marker_probe_array
*old
;
743 entry
= get_ust_marker(channel
, name
);
746 entry
= add_ust_marker(channel
, name
, format
);
748 ret
= PTR_ERR(entry
);
751 ret
= ltt_channels_register(channel
);
753 goto error_remove_ust_marker
;
754 ret
= ltt_channels_get_index_from_name(channel
);
756 goto error_unregister_channel
;
757 entry
->channel_id
= ret
;
758 ret
= ltt_channels_get_event_id(channel
, name
);
760 goto error_unregister_channel
;
761 entry
->event_id
= ret
;
763 __ust_marker(metadata
, core_marker_id
, NULL
,
764 "channel %s name %s event_id %hu "
765 "int #1u%zu long #1u%zu pointer #1u%zu "
766 "size_t #1u%zu alignment #1u%u",
767 channel
, name
, entry
->event_id
,
768 sizeof(int), sizeof(long), sizeof(void *),
769 sizeof(size_t), ltt_get_alignment());
772 ret
= ust_marker_set_format(entry
, format
);
773 else if (strcmp(entry
->format
, format
))
779 old
= ust_marker_entry_add_probe(entry
, probe
, probe_private
);
783 goto error_unregister_channel
;
789 /* Activate ust_marker if necessary */
790 ust_marker_update_probes();
794 free_old_closure(&old
->rcu
);
798 error_unregister_channel
:
799 ret_err
= ltt_channels_unregister(channel
);
801 error_remove_ust_marker
:
802 ret_err
= remove_ust_marker(channel
, name
);
810 * ust_marker_probe_unregister - Disconnect a probe from a ust_marker
811 * @channel: ust_marker channel
812 * @name: ust_marker name
813 * @probe: probe function pointer
814 * @probe_private: probe private data
816 * Returns the private data given to ust_marker_probe_register, or an ERR_PTR().
817 * We do not need to call a synchronize_sched to make sure the probes have
818 * finished running before doing a module unload, because the module unload
819 * itself uses stop_machine(), which insures that every preempt disabled section
822 int ust_marker_probe_unregister(const char *channel
, const char *name
,
823 ust_marker_probe_func
*probe
, void *probe_private
)
825 struct ust_marker_entry
*entry
;
826 struct ust_marker_probe_array
*old
;
830 entry
= get_ust_marker(channel
, name
);
835 old
= ust_marker_entry_remove_probe(entry
, probe
, probe_private
);
838 ust_marker_update_probes();
842 free_old_closure(&old
->rcu
);
851 static struct ust_marker_entry
*
852 get_ust_marker_from_private_data(ust_marker_probe_func
*probe
,
855 struct ust_marker_entry
*entry
;
857 struct cds_hlist_head
*head
;
858 struct cds_hlist_node
*node
;
860 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
861 head
= &ust_marker_table
[i
];
862 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
864 if (entry
->single
.func
== probe
865 && entry
->single
.probe_private
869 struct ust_marker_probe_array
*closure
;
870 closure
= entry
->multi
;
871 for (i
= 0; closure
->c
[i
].func
; i
++) {
872 if (closure
->c
[i
].func
== probe
&&
873 closure
->c
[i
].probe_private
884 * ust_marker_probe_unregister_private_data - Disconnect a probe from a ust_marker
885 * @probe: probe function
886 * @probe_private: probe private data
888 * Unregister a probe by providing the registered private data.
889 * Only removes the first ust_marker found in hash table.
890 * Return 0 on success or error value.
891 * We do not need to call a synchronize_sched to make sure the probes have
892 * finished running before doing a module unload, because the module unload
893 * itself uses stop_machine(), which insures that every preempt disabled section
896 int ust_marker_probe_unregister_private_data(ust_marker_probe_func
*probe
,
899 struct ust_marker_entry
*entry
;
901 struct ust_marker_probe_array
*old
;
902 char *channel
= NULL
, *name
= NULL
;
905 entry
= get_ust_marker_from_private_data(probe
, probe_private
);
910 old
= ust_marker_entry_remove_probe(entry
, NULL
, probe_private
);
911 channel
= strdup(entry
->channel
);
912 name
= strdup(entry
->name
);
913 /* Ignore busy error message */
914 remove_ust_marker(channel
, name
);
917 ust_marker_update_probes();
921 free_old_closure(&old
->rcu
);
934 * ust_marker_get_private_data - Get a ust_marker's probe private data
935 * @channel: ust_marker channel
936 * @name: ust_marker name
937 * @probe: probe to match
938 * @num: get the nth matching probe's private data
940 * Returns the nth private data pointer (starting from 0) matching, or an
942 * Returns the private data pointer, or an ERR_PTR.
943 * The private data pointer should _only_ be dereferenced if the caller is the
944 * owner of the data, or its content could vanish. This is mostly used to
945 * confirm that a caller is the owner of a registered probe.
947 void *ust_marker_get_private_data(const char *channel
, const char *name
,
948 ust_marker_probe_func
*probe
, int num
)
950 struct cds_hlist_head
*head
;
951 struct cds_hlist_node
*node
;
952 struct ust_marker_entry
*e
;
953 size_t channel_len
= strlen(channel
) + 1;
954 size_t name_len
= strlen(name
) + 1;
958 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
959 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
960 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
961 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
963 if (num
== 0 && e
->single
.func
== probe
)
964 return e
->single
.probe_private
;
966 struct ust_marker_probe_array
*closure
;
969 for (i
= 0; closure
->c
[i
].func
; i
++) {
970 if (closure
->c
[i
].func
!= probe
)
973 return closure
->c
[i
].probe_private
;
979 return ERR_PTR(-ENOENT
);
983 * ust_marker_get_iter_range - Get a next ust_marker iterator given a range.
984 * @ust_marker: current ust_marker (in), next ust_marker (out)
985 * @begin: beginning of the range
986 * @end: end of the range
988 * Returns whether a next ust_marker has been found (1) or not (0).
989 * Will return the first ust_marker in the range if the input ust_marker is NULL.
990 * Called with markers mutex held.
993 int ust_marker_get_iter_range(struct ust_marker
* const **ust_marker
,
994 struct ust_marker
* const *begin
,
995 struct ust_marker
* const *end
)
997 if (!*ust_marker
&& begin
!= end
)
999 while (*ust_marker
>= begin
&& *ust_marker
< end
) {
1001 (*ust_marker
)++; /* skip dummy */
1009 * Returns 0 if current not found.
1010 * Returns 1 if current found.
1011 * Called with markers mutex held.
1014 int lib_get_iter_ust_marker(struct ust_marker_iter
*iter
)
1016 struct ust_marker_lib
*iter_lib
;
1019 cds_list_for_each_entry(iter_lib
, &ust_marker_libs
, list
) {
1020 if (iter_lib
< iter
->lib
)
1022 else if (iter_lib
> iter
->lib
)
1023 iter
->ust_marker
= NULL
;
1024 found
= ust_marker_get_iter_range(&iter
->ust_marker
,
1025 iter_lib
->ust_marker_start
,
1026 iter_lib
->ust_marker_start
+ iter_lib
->ust_marker_count
);
1028 iter
->lib
= iter_lib
;
1035 /* Called with markers mutex held. */
1036 static void ust_marker_get_iter(struct ust_marker_iter
*iter
)
1040 found
= lib_get_iter_ust_marker(iter
);
1042 ust_marker_iter_reset(iter
);
1045 void ust_marker_iter_start(struct ust_marker_iter
*iter
)
1048 ust_marker_get_iter(iter
);
1051 /* Called with markers mutex held. */
1052 void ust_marker_iter_next(struct ust_marker_iter
*iter
)
1056 * iter->ust_marker may be invalid because we blindly incremented it.
1057 * Make sure it is valid by marshalling on the ust_marker, getting the
1058 * ust_marker from following modules if necessary.
1060 ust_marker_get_iter(iter
);
1063 void ust_marker_iter_stop(struct ust_marker_iter
*iter
)
1065 unlock_ust_marker();
1068 void ust_marker_iter_reset(struct ust_marker_iter
*iter
)
1071 iter
->ust_marker
= NULL
;
1074 void ltt_dump_ust_marker_state(struct ust_trace
*trace
)
1076 struct ust_marker_entry
*entry
;
1077 struct ltt_probe_private_data call_data
;
1078 struct cds_hlist_head
*head
;
1079 struct cds_hlist_node
*node
;
1083 call_data
.trace
= trace
;
1084 call_data
.serializer
= NULL
;
1086 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
1087 head
= &ust_marker_table
[i
];
1088 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
1089 __ust_marker(metadata
, core_marker_id
,
1091 "channel %s name %s event_id %hu "
1092 "int #1u%zu long #1u%zu pointer #1u%zu "
1093 "size_t #1u%zu alignment #1u%u",
1097 sizeof(int), sizeof(long),
1098 sizeof(void *), sizeof(size_t),
1099 ltt_get_alignment());
1101 __ust_marker(metadata
,
1104 "channel %s name %s format %s",
1110 unlock_ust_marker();
1113 void ust_marker_set_new_ust_marker_cb(void (*cb
)(struct ust_marker
*))
1115 new_ust_marker_cb
= cb
;
1118 static void new_ust_marker(struct ust_marker
* const *start
,
1119 struct ust_marker
* const *end
)
1121 if (new_ust_marker_cb
) {
1122 struct ust_marker
* const *m
;
1124 for (m
= start
; m
< end
; m
++) {
1126 new_ust_marker_cb(*m
);
1131 int ust_marker_register_lib(struct ust_marker
* const *ust_marker_start
,
1132 int ust_marker_count
)
1134 struct ust_marker_lib
*pl
, *iter
;
1136 pl
= (struct ust_marker_lib
*) zmalloc(sizeof(struct ust_marker_lib
));
1138 pl
->ust_marker_start
= ust_marker_start
;
1139 pl
->ust_marker_count
= ust_marker_count
;
1144 * We sort the libs by struct lib pointer address.
1146 cds_list_for_each_entry_reverse(iter
, &ust_marker_libs
, list
) {
1147 BUG_ON(iter
== pl
); /* Should never be in the list twice */
1149 /* We belong to the location right after iter. */
1150 cds_list_add(&pl
->list
, &iter
->list
);
1154 /* We should be added at the head of the list */
1155 cds_list_add(&pl
->list
, &ust_marker_libs
);
1157 unlock_ust_marker();
1159 new_ust_marker(ust_marker_start
, ust_marker_start
+ ust_marker_count
);
1161 /* TODO: update just the loaded lib */
1162 lib_update_ust_marker();
1164 DBG("just registered a ust_marker section from %p and having %d ust_marker (minus dummy ust_marker)", ust_marker_start
, ust_marker_count
);
1169 int ust_marker_unregister_lib(struct ust_marker
* const *ust_marker_start
)
1171 struct ust_marker_lib
*lib
;
1174 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
) {
1175 if(lib
->ust_marker_start
== ust_marker_start
) {
1176 struct ust_marker_lib
*lib2free
= lib
;
1177 cds_list_del(&lib
->list
);
1182 unlock_ust_marker();
1187 void __attribute__((constructor
)) init_ust_marker(void)
1191 ust_marker_register_lib(__start___ust_marker_ptrs
,
1192 __stop___ust_marker_ptrs
1193 - __start___ust_marker_ptrs
);
1198 void __attribute__((destructor
)) destroy_ust_marker(void)
1200 ust_marker_unregister_lib(__start___ust_marker_ptrs
);