Commit | Line | Data |
---|---|---|
68c1021b PMF |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
34e4b7db PMF |
4 | * This library is free software; you can redistribute it and/or |
5 | * modify it under the terms of the GNU Lesser General Public | |
6 | * License as published by the Free Software Foundation; either | |
7 | * version 2.1 of the License, or (at your option) any later version. | |
68c1021b | 8 | * |
34e4b7db | 9 | * This library is distributed in the hope that it will be useful, |
68c1021b | 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
34e4b7db PMF |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
12 | * Lesser General Public License for more details. | |
68c1021b | 13 | * |
34e4b7db PMF |
14 | * You should have received a copy of the GNU Lesser General Public |
15 | * License along with this library; if not, write to the Free Software | |
16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
68c1021b | 17 | */ |
59b161cd PMF |
18 | //ust// #include <linux/module.h> |
19 | //ust// #include <linux/mutex.h> | |
20 | //ust// #include <linux/types.h> | |
b4512257 PMF |
21 | //#include "jhash.h" |
22 | //#include "list.h" | |
23 | //#include "rcupdate.h" | |
59b161cd PMF |
24 | //ust// #include <linux/marker.h> |
25 | #include <errno.h> | |
26 | //ust// #include <linux/slab.h> | |
27 | //ust// #include <linux/immediate.h> | |
28 | //ust// #include <linux/sched.h> | |
29 | //ust// #include <linux/uaccess.h> | |
30 | //ust// #include <linux/user_marker.h> | |
31 | //ust// #include <linux/ltt-tracer.h> | |
32 | ||
769d0157 | 33 | #define _LGPL_SOURCE |
b7ea1a1c | 34 | #include <urcu-bp.h> |
769d0157 | 35 | |
fbca6b62 | 36 | #include <ust/kernelcompat.h> |
769d0157 | 37 | |
93d0f2ea | 38 | #include <ust/marker.h> |
59b161cd PMF |
39 | #include "usterr.h" |
40 | #include "channels.h" | |
41 | #include "tracercore.h" | |
c93858f1 | 42 | #include "tracer.h" |
68c1021b | 43 | |
636ca5d6 PMF |
44 | __thread long ust_reg_stack[500]; |
45 | volatile __thread long *ust_reg_stack_ptr = (long *) 0; | |
46 | ||
c463904d PMF |
47 | extern struct marker __start___markers[] __attribute__((visibility("hidden"))); |
48 | extern struct marker __stop___markers[] __attribute__((visibility("hidden"))); | |
defa46a7 PMF |
49 | |
50 | #ifdef CONFIG_UST_GDB_INTEGRATION | |
3ea1e2fc PMF |
51 | extern struct marker_addr __start___marker_addr[] __attribute__((visibility("hidden"))); |
52 | extern struct marker_addr __stop___marker_addr[] __attribute__((visibility("hidden"))); | |
defa46a7 | 53 | #endif |
68c1021b PMF |
54 | |
55 | /* Set to 1 to enable marker debug output */ | |
56 | static const int marker_debug; | |
57 | ||
58 | /* | |
59 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin | |
60 | * and module markers and the hash table. | |
61 | */ | |
62 | static DEFINE_MUTEX(markers_mutex); | |
63 | ||
772030fe PMF |
64 | static LIST_HEAD(libs); |
65 | ||
66 | ||
68c1021b PMF |
67 | void lock_markers(void) |
68 | { | |
69 | mutex_lock(&markers_mutex); | |
70 | } | |
71 | ||
72 | void unlock_markers(void) | |
73 | { | |
74 | mutex_unlock(&markers_mutex); | |
75 | } | |
76 | ||
77 | /* | |
78 | * Marker hash table, containing the active markers. | |
79 | * Protected by module_mutex. | |
80 | */ | |
81 | #define MARKER_HASH_BITS 6 | |
82 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
83 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
84 | ||
85 | /* | |
86 | * Note about RCU : | |
87 | * It is used to make sure every handler has finished using its private data | |
88 | * between two consecutive operation (add or remove) on a given marker. It is | |
89 | * also used to delay the free of multiple probes array until a quiescent state | |
90 | * is reached. | |
91 | * marker entries modifications are protected by the markers_mutex. | |
92 | */ | |
93 | struct marker_entry { | |
94 | struct hlist_node hlist; | |
95 | char *format; | |
96 | char *name; | |
97 | /* Probe wrapper */ | |
75667d04 | 98 | void (*call)(const struct marker *mdata, void *call_private, struct registers *regs, ...); |
68c1021b PMF |
99 | struct marker_probe_closure single; |
100 | struct marker_probe_closure *multi; | |
101 | int refcount; /* Number of times armed. 0 if disarmed. */ | |
102 | struct rcu_head rcu; | |
103 | void *oldptr; | |
104 | int rcu_pending; | |
105 | u16 channel_id; | |
106 | u16 event_id; | |
107 | unsigned char ptype:1; | |
108 | unsigned char format_allocated:1; | |
109 | char channel[0]; /* Contains channel'\0'name'\0'format'\0' */ | |
110 | }; | |
111 | ||
112 | #ifdef CONFIG_MARKERS_USERSPACE | |
113 | static void marker_update_processes(void); | |
114 | #else | |
115 | static void marker_update_processes(void) | |
116 | { | |
117 | } | |
118 | #endif | |
119 | ||
120 | /** | |
121 | * __mark_empty_function - Empty probe callback | |
122 | * @mdata: marker data | |
123 | * @probe_private: probe private data | |
124 | * @call_private: call site private data | |
125 | * @fmt: format string | |
126 | * @...: variable argument list | |
127 | * | |
128 | * Empty callback provided as a probe to the markers. By providing this to a | |
129 | * disabled marker, we make sure the execution flow is always valid even | |
130 | * though the function pointer change and the marker enabling are two distinct | |
131 | * operations that modifies the execution flow of preemptible code. | |
132 | */ | |
133 | notrace void __mark_empty_function(const struct marker *mdata, | |
75667d04 | 134 | void *probe_private, struct registers *regs, void *call_private, const char *fmt, va_list *args) |
68c1021b PMF |
135 | { |
136 | } | |
59b161cd | 137 | //ust// EXPORT_SYMBOL_GPL(__mark_empty_function); |
68c1021b PMF |
138 | |
139 | /* | |
140 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
141 | * @mdata: pointer of type struct marker | |
142 | * @call_private: caller site private data | |
143 | * @...: Variable argument list. | |
144 | * | |
145 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
146 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
147 | * rcu_dereference() for the pointer read. | |
148 | */ | |
149 | notrace void marker_probe_cb(const struct marker *mdata, | |
75667d04 | 150 | void *call_private, struct registers *regs, ...) |
68c1021b PMF |
151 | { |
152 | va_list args; | |
153 | char ptype; | |
154 | ||
155 | /* | |
156 | * rcu_read_lock_sched does two things : disabling preemption to make | |
157 | * sure the teardown of the callbacks can be done correctly when they | |
158 | * are in modules and they insure RCU read coherency. | |
159 | */ | |
59b161cd | 160 | //ust// rcu_read_lock_sched_notrace(); |
68c1021b PMF |
161 | ptype = mdata->ptype; |
162 | if (likely(!ptype)) { | |
163 | marker_probe_func *func; | |
164 | /* Must read the ptype before ptr. They are not data dependant, | |
165 | * so we put an explicit smp_rmb() here. */ | |
166 | smp_rmb(); | |
167 | func = mdata->single.func; | |
168 | /* Must read the ptr before private data. They are not data | |
169 | * dependant, so we put an explicit smp_rmb() here. */ | |
170 | smp_rmb(); | |
75667d04 PMF |
171 | va_start(args, regs); |
172 | func(mdata, mdata->single.probe_private, regs, call_private, | |
68c1021b PMF |
173 | mdata->format, &args); |
174 | va_end(args); | |
175 | } else { | |
176 | struct marker_probe_closure *multi; | |
177 | int i; | |
178 | /* | |
179 | * Read mdata->ptype before mdata->multi. | |
180 | */ | |
181 | smp_rmb(); | |
182 | multi = mdata->multi; | |
183 | /* | |
184 | * multi points to an array, therefore accessing the array | |
185 | * depends on reading multi. However, even in this case, | |
186 | * we must insure that the pointer is read _before_ the array | |
187 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
188 | * in the fast path, so put the explicit barrier here. | |
189 | */ | |
190 | smp_read_barrier_depends(); | |
191 | for (i = 0; multi[i].func; i++) { | |
75667d04 | 192 | va_start(args, regs); |
68c1021b | 193 | multi[i].func(mdata, multi[i].probe_private, |
75667d04 | 194 | regs, call_private, mdata->format, &args); |
68c1021b PMF |
195 | va_end(args); |
196 | } | |
197 | } | |
59b161cd | 198 | //ust// rcu_read_unlock_sched_notrace(); |
68c1021b | 199 | } |
59b161cd | 200 | //ust// EXPORT_SYMBOL_GPL(marker_probe_cb); |
68c1021b PMF |
201 | |
202 | /* | |
203 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
204 | * @mdata: pointer of type struct marker | |
205 | * @call_private: caller site private data | |
206 | * @...: Variable argument list. | |
207 | * | |
208 | * Should be connected to markers "MARK_NOARGS". | |
209 | */ | |
210 | static notrace void marker_probe_cb_noarg(const struct marker *mdata, | |
75667d04 | 211 | void *call_private, struct registers *regs, ...) |
68c1021b PMF |
212 | { |
213 | va_list args; /* not initialized */ | |
214 | char ptype; | |
215 | ||
59b161cd | 216 | //ust// rcu_read_lock_sched_notrace(); |
68c1021b PMF |
217 | ptype = mdata->ptype; |
218 | if (likely(!ptype)) { | |
219 | marker_probe_func *func; | |
220 | /* Must read the ptype before ptr. They are not data dependant, | |
221 | * so we put an explicit smp_rmb() here. */ | |
222 | smp_rmb(); | |
223 | func = mdata->single.func; | |
224 | /* Must read the ptr before private data. They are not data | |
225 | * dependant, so we put an explicit smp_rmb() here. */ | |
226 | smp_rmb(); | |
75667d04 | 227 | func(mdata, mdata->single.probe_private, regs, call_private, |
68c1021b PMF |
228 | mdata->format, &args); |
229 | } else { | |
230 | struct marker_probe_closure *multi; | |
231 | int i; | |
232 | /* | |
233 | * Read mdata->ptype before mdata->multi. | |
234 | */ | |
235 | smp_rmb(); | |
236 | multi = mdata->multi; | |
237 | /* | |
238 | * multi points to an array, therefore accessing the array | |
239 | * depends on reading multi. However, even in this case, | |
240 | * we must insure that the pointer is read _before_ the array | |
241 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
242 | * in the fast path, so put the explicit barrier here. | |
243 | */ | |
244 | smp_read_barrier_depends(); | |
245 | for (i = 0; multi[i].func; i++) | |
75667d04 | 246 | multi[i].func(mdata, multi[i].probe_private, regs, |
68c1021b PMF |
247 | call_private, mdata->format, &args); |
248 | } | |
59b161cd | 249 | //ust// rcu_read_unlock_sched_notrace(); |
68c1021b PMF |
250 | } |
251 | ||
252 | static void free_old_closure(struct rcu_head *head) | |
253 | { | |
254 | struct marker_entry *entry = container_of(head, | |
255 | struct marker_entry, rcu); | |
256 | kfree(entry->oldptr); | |
257 | /* Make sure we free the data before setting the pending flag to 0 */ | |
258 | smp_wmb(); | |
259 | entry->rcu_pending = 0; | |
260 | } | |
261 | ||
262 | static void debug_print_probes(struct marker_entry *entry) | |
263 | { | |
264 | int i; | |
265 | ||
266 | if (!marker_debug) | |
267 | return; | |
268 | ||
269 | if (!entry->ptype) { | |
270 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
271 | entry->single.func, | |
272 | entry->single.probe_private); | |
273 | } else { | |
274 | for (i = 0; entry->multi[i].func; i++) | |
275 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
276 | entry->multi[i].func, | |
277 | entry->multi[i].probe_private); | |
278 | } | |
279 | } | |
280 | ||
281 | static struct marker_probe_closure * | |
282 | marker_entry_add_probe(struct marker_entry *entry, | |
283 | marker_probe_func *probe, void *probe_private) | |
284 | { | |
285 | int nr_probes = 0; | |
286 | struct marker_probe_closure *old, *new; | |
287 | ||
288 | WARN_ON(!probe); | |
289 | ||
290 | debug_print_probes(entry); | |
291 | old = entry->multi; | |
292 | if (!entry->ptype) { | |
293 | if (entry->single.func == probe && | |
294 | entry->single.probe_private == probe_private) | |
295 | return ERR_PTR(-EBUSY); | |
296 | if (entry->single.func == __mark_empty_function) { | |
297 | /* 0 -> 1 probes */ | |
298 | entry->single.func = probe; | |
299 | entry->single.probe_private = probe_private; | |
300 | entry->refcount = 1; | |
301 | entry->ptype = 0; | |
302 | debug_print_probes(entry); | |
303 | return NULL; | |
304 | } else { | |
305 | /* 1 -> 2 probes */ | |
306 | nr_probes = 1; | |
307 | old = NULL; | |
308 | } | |
309 | } else { | |
310 | /* (N -> N+1), (N != 0, 1) probes */ | |
311 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
312 | if (old[nr_probes].func == probe | |
313 | && old[nr_probes].probe_private | |
314 | == probe_private) | |
315 | return ERR_PTR(-EBUSY); | |
316 | } | |
317 | /* + 2 : one for new probe, one for NULL func */ | |
318 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
319 | GFP_KERNEL); | |
320 | if (new == NULL) | |
321 | return ERR_PTR(-ENOMEM); | |
322 | if (!old) | |
323 | new[0] = entry->single; | |
324 | else | |
325 | memcpy(new, old, | |
326 | nr_probes * sizeof(struct marker_probe_closure)); | |
327 | new[nr_probes].func = probe; | |
328 | new[nr_probes].probe_private = probe_private; | |
329 | entry->refcount = nr_probes + 1; | |
330 | entry->multi = new; | |
331 | entry->ptype = 1; | |
332 | debug_print_probes(entry); | |
333 | return old; | |
334 | } | |
335 | ||
336 | static struct marker_probe_closure * | |
337 | marker_entry_remove_probe(struct marker_entry *entry, | |
338 | marker_probe_func *probe, void *probe_private) | |
339 | { | |
340 | int nr_probes = 0, nr_del = 0, i; | |
341 | struct marker_probe_closure *old, *new; | |
342 | ||
343 | old = entry->multi; | |
344 | ||
345 | debug_print_probes(entry); | |
346 | if (!entry->ptype) { | |
347 | /* 0 -> N is an error */ | |
348 | WARN_ON(entry->single.func == __mark_empty_function); | |
349 | /* 1 -> 0 probes */ | |
350 | WARN_ON(probe && entry->single.func != probe); | |
351 | WARN_ON(entry->single.probe_private != probe_private); | |
352 | entry->single.func = __mark_empty_function; | |
353 | entry->refcount = 0; | |
354 | entry->ptype = 0; | |
355 | debug_print_probes(entry); | |
356 | return NULL; | |
357 | } else { | |
358 | /* (N -> M), (N > 1, M >= 0) probes */ | |
359 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
360 | if ((!probe || old[nr_probes].func == probe) | |
361 | && old[nr_probes].probe_private | |
362 | == probe_private) | |
363 | nr_del++; | |
364 | } | |
365 | } | |
366 | ||
367 | if (nr_probes - nr_del == 0) { | |
368 | /* N -> 0, (N > 1) */ | |
369 | entry->single.func = __mark_empty_function; | |
370 | entry->refcount = 0; | |
371 | entry->ptype = 0; | |
372 | } else if (nr_probes - nr_del == 1) { | |
373 | /* N -> 1, (N > 1) */ | |
374 | for (i = 0; old[i].func; i++) | |
375 | if ((probe && old[i].func != probe) || | |
376 | old[i].probe_private != probe_private) | |
377 | entry->single = old[i]; | |
378 | entry->refcount = 1; | |
379 | entry->ptype = 0; | |
380 | } else { | |
381 | int j = 0; | |
382 | /* N -> M, (N > 1, M > 1) */ | |
383 | /* + 1 for NULL */ | |
384 | new = kzalloc((nr_probes - nr_del + 1) | |
385 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
386 | if (new == NULL) | |
387 | return ERR_PTR(-ENOMEM); | |
388 | for (i = 0; old[i].func; i++) | |
389 | if ((probe && old[i].func != probe) || | |
390 | old[i].probe_private != probe_private) | |
391 | new[j++] = old[i]; | |
392 | entry->refcount = nr_probes - nr_del; | |
393 | entry->ptype = 1; | |
394 | entry->multi = new; | |
395 | } | |
396 | debug_print_probes(entry); | |
397 | return old; | |
398 | } | |
399 | ||
400 | /* | |
401 | * Get marker if the marker is present in the marker hash table. | |
402 | * Must be called with markers_mutex held. | |
403 | * Returns NULL if not present. | |
404 | */ | |
405 | static struct marker_entry *get_marker(const char *channel, const char *name) | |
406 | { | |
407 | struct hlist_head *head; | |
408 | struct hlist_node *node; | |
409 | struct marker_entry *e; | |
410 | size_t channel_len = strlen(channel) + 1; | |
411 | size_t name_len = strlen(name) + 1; | |
412 | u32 hash; | |
413 | ||
414 | hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); | |
415 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
416 | hlist_for_each_entry(e, node, head, hlist) { | |
417 | if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) | |
418 | return e; | |
419 | } | |
420 | return NULL; | |
421 | } | |
422 | ||
423 | /* | |
424 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
425 | * held. | |
426 | */ | |
427 | static struct marker_entry *add_marker(const char *channel, const char *name, | |
428 | const char *format) | |
429 | { | |
430 | struct hlist_head *head; | |
431 | struct hlist_node *node; | |
432 | struct marker_entry *e; | |
433 | size_t channel_len = strlen(channel) + 1; | |
434 | size_t name_len = strlen(name) + 1; | |
435 | size_t format_len = 0; | |
436 | u32 hash; | |
437 | ||
438 | hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); | |
439 | if (format) | |
440 | format_len = strlen(format) + 1; | |
441 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
442 | hlist_for_each_entry(e, node, head, hlist) { | |
443 | if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { | |
444 | printk(KERN_NOTICE | |
445 | "Marker %s.%s busy\n", channel, name); | |
446 | return ERR_PTR(-EBUSY); /* Already there */ | |
447 | } | |
448 | } | |
449 | /* | |
450 | * Using kmalloc here to allocate a variable length element. Could | |
451 | * cause some memory fragmentation if overused. | |
452 | */ | |
453 | e = kmalloc(sizeof(struct marker_entry) | |
454 | + channel_len + name_len + format_len, | |
455 | GFP_KERNEL); | |
456 | if (!e) | |
457 | return ERR_PTR(-ENOMEM); | |
458 | memcpy(e->channel, channel, channel_len); | |
459 | e->name = &e->channel[channel_len]; | |
460 | memcpy(e->name, name, name_len); | |
461 | if (format) { | |
462 | e->format = &e->name[channel_len + name_len]; | |
463 | memcpy(e->format, format, format_len); | |
464 | if (strcmp(e->format, MARK_NOARGS) == 0) | |
465 | e->call = marker_probe_cb_noarg; | |
466 | else | |
467 | e->call = marker_probe_cb; | |
468 | trace_mark(metadata, core_marker_format, | |
469 | "channel %s name %s format %s", | |
470 | e->channel, e->name, e->format); | |
471 | } else { | |
472 | e->format = NULL; | |
473 | e->call = marker_probe_cb; | |
474 | } | |
475 | e->single.func = __mark_empty_function; | |
476 | e->single.probe_private = NULL; | |
477 | e->multi = NULL; | |
478 | e->ptype = 0; | |
479 | e->format_allocated = 0; | |
480 | e->refcount = 0; | |
481 | e->rcu_pending = 0; | |
482 | hlist_add_head(&e->hlist, head); | |
483 | return e; | |
484 | } | |
485 | ||
486 | /* | |
487 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
488 | * held. | |
489 | */ | |
490 | static int remove_marker(const char *channel, const char *name) | |
491 | { | |
492 | struct hlist_head *head; | |
493 | struct hlist_node *node; | |
494 | struct marker_entry *e; | |
495 | int found = 0; | |
496 | size_t channel_len = strlen(channel) + 1; | |
497 | size_t name_len = strlen(name) + 1; | |
498 | u32 hash; | |
499 | int ret; | |
500 | ||
501 | hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); | |
502 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
503 | hlist_for_each_entry(e, node, head, hlist) { | |
504 | if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { | |
505 | found = 1; | |
506 | break; | |
507 | } | |
508 | } | |
509 | if (!found) | |
510 | return -ENOENT; | |
511 | if (e->single.func != __mark_empty_function) | |
512 | return -EBUSY; | |
513 | hlist_del(&e->hlist); | |
514 | if (e->format_allocated) | |
515 | kfree(e->format); | |
516 | ret = ltt_channels_unregister(e->channel); | |
517 | WARN_ON(ret); | |
518 | /* Make sure the call_rcu has been executed */ | |
6cb88bc0 PMF |
519 | //ust// if (e->rcu_pending) |
520 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
521 | kfree(e); |
522 | return 0; | |
523 | } | |
524 | ||
525 | /* | |
526 | * Set the mark_entry format to the format found in the element. | |
527 | */ | |
528 | static int marker_set_format(struct marker_entry *entry, const char *format) | |
529 | { | |
530 | entry->format = kstrdup(format, GFP_KERNEL); | |
531 | if (!entry->format) | |
532 | return -ENOMEM; | |
533 | entry->format_allocated = 1; | |
534 | ||
535 | trace_mark(metadata, core_marker_format, | |
536 | "channel %s name %s format %s", | |
537 | entry->channel, entry->name, entry->format); | |
538 | return 0; | |
539 | } | |
540 | ||
541 | /* | |
542 | * Sets the probe callback corresponding to one marker. | |
543 | */ | |
544 | static int set_marker(struct marker_entry *entry, struct marker *elem, | |
545 | int active) | |
546 | { | |
547 | int ret = 0; | |
548 | WARN_ON(strcmp(entry->name, elem->name) != 0); | |
549 | ||
550 | if (entry->format) { | |
551 | if (strcmp(entry->format, elem->format) != 0) { | |
552 | printk(KERN_NOTICE | |
553 | "Format mismatch for probe %s " | |
554 | "(%s), marker (%s)\n", | |
555 | entry->name, | |
556 | entry->format, | |
557 | elem->format); | |
558 | return -EPERM; | |
559 | } | |
560 | } else { | |
561 | ret = marker_set_format(entry, elem->format); | |
562 | if (ret) | |
563 | return ret; | |
564 | } | |
565 | ||
566 | /* | |
567 | * probe_cb setup (statically known) is done here. It is | |
568 | * asynchronous with the rest of execution, therefore we only | |
569 | * pass from a "safe" callback (with argument) to an "unsafe" | |
570 | * callback (does not set arguments). | |
571 | */ | |
572 | elem->call = entry->call; | |
573 | elem->channel_id = entry->channel_id; | |
574 | elem->event_id = entry->event_id; | |
575 | /* | |
576 | * Sanity check : | |
577 | * We only update the single probe private data when the ptr is | |
578 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
579 | */ | |
580 | WARN_ON(elem->single.func != __mark_empty_function | |
581 | && elem->single.probe_private != entry->single.probe_private | |
582 | && !elem->ptype); | |
583 | elem->single.probe_private = entry->single.probe_private; | |
584 | /* | |
585 | * Make sure the private data is valid when we update the | |
586 | * single probe ptr. | |
587 | */ | |
588 | smp_wmb(); | |
589 | elem->single.func = entry->single.func; | |
590 | /* | |
591 | * We also make sure that the new probe callbacks array is consistent | |
592 | * before setting a pointer to it. | |
593 | */ | |
594 | rcu_assign_pointer(elem->multi, entry->multi); | |
595 | /* | |
596 | * Update the function or multi probe array pointer before setting the | |
597 | * ptype. | |
598 | */ | |
599 | smp_wmb(); | |
600 | elem->ptype = entry->ptype; | |
601 | ||
59b161cd PMF |
602 | //ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) { |
603 | //ust// WARN_ON(!elem->tp_cb); | |
604 | //ust// /* | |
605 | //ust// * It is ok to directly call the probe registration because type | |
606 | //ust// * checking has been done in the __trace_mark_tp() macro. | |
607 | //ust// */ | |
608 | //ust// | |
609 | //ust// if (active) { | |
610 | //ust// /* | |
611 | //ust// * try_module_get should always succeed because we hold | |
612 | //ust// * markers_mutex to get the tp_cb address. | |
613 | //ust// */ | |
614 | //ust// ret = try_module_get(__module_text_address( | |
615 | //ust// (unsigned long)elem->tp_cb)); | |
616 | //ust// BUG_ON(!ret); | |
617 | //ust// ret = tracepoint_probe_register_noupdate( | |
618 | //ust// elem->tp_name, | |
619 | //ust// elem->tp_cb); | |
620 | //ust// } else { | |
621 | //ust// ret = tracepoint_probe_unregister_noupdate( | |
622 | //ust// elem->tp_name, | |
623 | //ust// elem->tp_cb); | |
624 | //ust// /* | |
625 | //ust// * tracepoint_probe_update_all() must be called | |
626 | //ust// * before the module containing tp_cb is unloaded. | |
627 | //ust// */ | |
628 | //ust// module_put(__module_text_address( | |
629 | //ust// (unsigned long)elem->tp_cb)); | |
630 | //ust// } | |
631 | //ust// } | |
68c1021b PMF |
632 | elem->state__imv = active; |
633 | ||
634 | return ret; | |
635 | } | |
636 | ||
637 | /* | |
638 | * Disable a marker and its probe callback. | |
639 | * Note: only waiting an RCU period after setting elem->call to the empty | |
640 | * function insures that the original callback is not used anymore. This insured | |
641 | * by rcu_read_lock_sched around the call site. | |
642 | */ | |
643 | static void disable_marker(struct marker *elem) | |
644 | { | |
772030fe PMF |
645 | //ust// int ret; |
646 | //ust// | |
647 | //ust// /* leave "call" as is. It is known statically. */ | |
59b161cd PMF |
648 | //ust// if (elem->tp_name && _imv_read(elem->state)) { |
649 | //ust// WARN_ON(!elem->tp_cb); | |
650 | //ust// /* | |
651 | //ust// * It is ok to directly call the probe registration because type | |
652 | //ust// * checking has been done in the __trace_mark_tp() macro. | |
653 | //ust// */ | |
654 | //ust// ret = tracepoint_probe_unregister_noupdate(elem->tp_name, | |
655 | //ust// elem->tp_cb); | |
656 | //ust// WARN_ON(ret); | |
657 | //ust// /* | |
658 | //ust// * tracepoint_probe_update_all() must be called | |
659 | //ust// * before the module containing tp_cb is unloaded. | |
660 | //ust// */ | |
661 | //ust// module_put(__module_text_address((unsigned long)elem->tp_cb)); | |
662 | //ust// } | |
68c1021b PMF |
663 | elem->state__imv = 0; |
664 | elem->single.func = __mark_empty_function; | |
665 | /* Update the function before setting the ptype */ | |
666 | smp_wmb(); | |
667 | elem->ptype = 0; /* single probe */ | |
668 | /* | |
669 | * Leave the private data and channel_id/event_id there, because removal | |
670 | * is racy and should be done only after an RCU period. These are never | |
671 | * used until the next initialization anyway. | |
672 | */ | |
673 | } | |
674 | ||
675 | /** | |
676 | * marker_update_probe_range - Update a probe range | |
677 | * @begin: beginning of the range | |
678 | * @end: end of the range | |
679 | * | |
680 | * Updates the probe callback corresponding to a range of markers. | |
681 | */ | |
682 | void marker_update_probe_range(struct marker *begin, | |
683 | struct marker *end) | |
684 | { | |
685 | struct marker *iter; | |
686 | struct marker_entry *mark_entry; | |
687 | ||
688 | mutex_lock(&markers_mutex); | |
689 | for (iter = begin; iter < end; iter++) { | |
690 | mark_entry = get_marker(iter->channel, iter->name); | |
691 | if (mark_entry) { | |
692 | set_marker(mark_entry, iter, !!mark_entry->refcount); | |
693 | /* | |
694 | * ignore error, continue | |
695 | */ | |
4db647c5 PMF |
696 | |
697 | /* This is added for UST. We emit a core_marker_id event | |
698 | * for markers that are already registered to a probe | |
699 | * upon library load. Otherwise, no core_marker_id will | |
700 | * be generated for these markers. Is this the right thing | |
701 | * to do? | |
702 | */ | |
703 | trace_mark(metadata, core_marker_id, | |
704 | "channel %s name %s event_id %hu " | |
705 | "int #1u%zu long #1u%zu pointer #1u%zu " | |
706 | "size_t #1u%zu alignment #1u%u", | |
707 | iter->channel, iter->name, mark_entry->event_id, | |
708 | sizeof(int), sizeof(long), sizeof(void *), | |
709 | sizeof(size_t), ltt_get_alignment()); | |
68c1021b PMF |
710 | } else { |
711 | disable_marker(iter); | |
712 | } | |
713 | } | |
714 | mutex_unlock(&markers_mutex); | |
715 | } | |
716 | ||
772030fe PMF |
717 | static void lib_update_markers(void) |
718 | { | |
719 | struct lib *lib; | |
720 | ||
721 | /* FIXME: we should probably take a mutex here on libs */ | |
722 | //ust// mutex_lock(&module_mutex); | |
723 | list_for_each_entry(lib, &libs, list) | |
724 | marker_update_probe_range(lib->markers_start, | |
725 | lib->markers_start + lib->markers_count); | |
726 | //ust// mutex_unlock(&module_mutex); | |
727 | } | |
728 | ||
68c1021b PMF |
729 | /* |
730 | * Update probes, removing the faulty probes. | |
731 | * | |
732 | * Internal callback only changed before the first probe is connected to it. | |
733 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
734 | * transitions. All other transitions will leave the old private data valid. | |
735 | * This makes the non-atomicity of the callback/private data updates valid. | |
736 | * | |
737 | * "special case" updates : | |
738 | * 0 -> 1 callback | |
739 | * 1 -> 0 callback | |
740 | * 1 -> 2 callbacks | |
741 | * 2 -> 1 callbacks | |
742 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
743 | * Site effect : marker_set_format may delete the marker entry (creating a | |
744 | * replacement). | |
745 | */ | |
746 | static void marker_update_probes(void) | |
747 | { | |
748 | /* Core kernel markers */ | |
c463904d | 749 | //ust// marker_update_probe_range(__start___markers, __stop___markers); |
68c1021b | 750 | /* Markers in modules. */ |
59b161cd | 751 | //ust// module_update_markers(); |
c463904d | 752 | lib_update_markers(); |
b6bf28ec | 753 | //ust// tracepoint_probe_update_all(); |
68c1021b PMF |
754 | /* Update immediate values */ |
755 | core_imv_update(); | |
474d745f | 756 | //ust// module_imv_update(); /* FIXME: need to port for libs? */ |
68c1021b PMF |
757 | marker_update_processes(); |
758 | } | |
759 | ||
760 | /** | |
761 | * marker_probe_register - Connect a probe to a marker | |
762 | * @channel: marker channel | |
763 | * @name: marker name | |
764 | * @format: format string | |
765 | * @probe: probe handler | |
766 | * @probe_private: probe private data | |
767 | * | |
768 | * private data must be a valid allocated memory address, or NULL. | |
769 | * Returns 0 if ok, error value on error. | |
770 | * The probe address must at least be aligned on the architecture pointer size. | |
771 | */ | |
772 | int marker_probe_register(const char *channel, const char *name, | |
773 | const char *format, marker_probe_func *probe, | |
774 | void *probe_private) | |
775 | { | |
776 | struct marker_entry *entry; | |
777 | int ret = 0, ret_err; | |
778 | struct marker_probe_closure *old; | |
779 | int first_probe = 0; | |
780 | ||
781 | mutex_lock(&markers_mutex); | |
782 | entry = get_marker(channel, name); | |
783 | if (!entry) { | |
784 | first_probe = 1; | |
785 | entry = add_marker(channel, name, format); | |
786 | if (IS_ERR(entry)) | |
787 | ret = PTR_ERR(entry); | |
788 | if (ret) | |
789 | goto end; | |
790 | ret = ltt_channels_register(channel); | |
791 | if (ret) | |
792 | goto error_remove_marker; | |
793 | ret = ltt_channels_get_index_from_name(channel); | |
794 | if (ret < 0) | |
795 | goto error_unregister_channel; | |
796 | entry->channel_id = ret; | |
797 | ret = ltt_channels_get_event_id(channel, name); | |
798 | if (ret < 0) | |
799 | goto error_unregister_channel; | |
800 | entry->event_id = ret; | |
801 | ret = 0; | |
802 | trace_mark(metadata, core_marker_id, | |
803 | "channel %s name %s event_id %hu " | |
804 | "int #1u%zu long #1u%zu pointer #1u%zu " | |
805 | "size_t #1u%zu alignment #1u%u", | |
806 | channel, name, entry->event_id, | |
807 | sizeof(int), sizeof(long), sizeof(void *), | |
808 | sizeof(size_t), ltt_get_alignment()); | |
809 | } else if (format) { | |
810 | if (!entry->format) | |
811 | ret = marker_set_format(entry, format); | |
812 | else if (strcmp(entry->format, format)) | |
813 | ret = -EPERM; | |
814 | if (ret) | |
815 | goto end; | |
816 | } | |
817 | ||
818 | /* | |
819 | * If we detect that a call_rcu is pending for this marker, | |
820 | * make sure it's executed now. | |
821 | */ | |
6cb88bc0 PMF |
822 | //ust// if (entry->rcu_pending) |
823 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
824 | old = marker_entry_add_probe(entry, probe, probe_private); |
825 | if (IS_ERR(old)) { | |
826 | ret = PTR_ERR(old); | |
827 | if (first_probe) | |
828 | goto error_unregister_channel; | |
829 | else | |
830 | goto end; | |
831 | } | |
832 | mutex_unlock(&markers_mutex); | |
833 | ||
79e36890 | 834 | /* Activate marker if necessary */ |
68c1021b PMF |
835 | marker_update_probes(); |
836 | ||
837 | mutex_lock(&markers_mutex); | |
838 | entry = get_marker(channel, name); | |
839 | if (!entry) | |
840 | goto end; | |
6cb88bc0 PMF |
841 | //ust// if (entry->rcu_pending) |
842 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
843 | entry->oldptr = old; |
844 | entry->rcu_pending = 1; | |
845 | /* write rcu_pending before calling the RCU callback */ | |
846 | smp_wmb(); | |
6cb88bc0 PMF |
847 | //ust// call_rcu_sched(&entry->rcu, free_old_closure); |
848 | synchronize_rcu(); free_old_closure(&entry->rcu); | |
68c1021b PMF |
849 | goto end; |
850 | ||
851 | error_unregister_channel: | |
852 | ret_err = ltt_channels_unregister(channel); | |
853 | WARN_ON(ret_err); | |
854 | error_remove_marker: | |
855 | ret_err = remove_marker(channel, name); | |
856 | WARN_ON(ret_err); | |
857 | end: | |
858 | mutex_unlock(&markers_mutex); | |
859 | return ret; | |
860 | } | |
59b161cd | 861 | //ust// EXPORT_SYMBOL_GPL(marker_probe_register); |
68c1021b PMF |
862 | |
863 | /** | |
864 | * marker_probe_unregister - Disconnect a probe from a marker | |
865 | * @channel: marker channel | |
866 | * @name: marker name | |
867 | * @probe: probe function pointer | |
868 | * @probe_private: probe private data | |
869 | * | |
870 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
871 | * We do not need to call a synchronize_sched to make sure the probes have | |
872 | * finished running before doing a module unload, because the module unload | |
873 | * itself uses stop_machine(), which insures that every preempt disabled section | |
874 | * have finished. | |
875 | */ | |
876 | int marker_probe_unregister(const char *channel, const char *name, | |
877 | marker_probe_func *probe, void *probe_private) | |
878 | { | |
879 | struct marker_entry *entry; | |
880 | struct marker_probe_closure *old; | |
881 | int ret = -ENOENT; | |
882 | ||
883 | mutex_lock(&markers_mutex); | |
884 | entry = get_marker(channel, name); | |
885 | if (!entry) | |
886 | goto end; | |
6cb88bc0 PMF |
887 | //ust// if (entry->rcu_pending) |
888 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
889 | old = marker_entry_remove_probe(entry, probe, probe_private); |
890 | mutex_unlock(&markers_mutex); | |
891 | ||
892 | marker_update_probes(); | |
893 | ||
894 | mutex_lock(&markers_mutex); | |
895 | entry = get_marker(channel, name); | |
896 | if (!entry) | |
897 | goto end; | |
6cb88bc0 PMF |
898 | //ust// if (entry->rcu_pending) |
899 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
900 | entry->oldptr = old; |
901 | entry->rcu_pending = 1; | |
902 | /* write rcu_pending before calling the RCU callback */ | |
903 | smp_wmb(); | |
6cb88bc0 PMF |
904 | //ust// call_rcu_sched(&entry->rcu, free_old_closure); |
905 | synchronize_rcu(); free_old_closure(&entry->rcu); | |
68c1021b PMF |
906 | remove_marker(channel, name); /* Ignore busy error message */ |
907 | ret = 0; | |
908 | end: | |
909 | mutex_unlock(&markers_mutex); | |
910 | return ret; | |
911 | } | |
59b161cd | 912 | //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister); |
68c1021b PMF |
913 | |
914 | static struct marker_entry * | |
915 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
916 | { | |
917 | struct marker_entry *entry; | |
918 | unsigned int i; | |
919 | struct hlist_head *head; | |
920 | struct hlist_node *node; | |
921 | ||
922 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { | |
923 | head = &marker_table[i]; | |
924 | hlist_for_each_entry(entry, node, head, hlist) { | |
925 | if (!entry->ptype) { | |
926 | if (entry->single.func == probe | |
927 | && entry->single.probe_private | |
928 | == probe_private) | |
929 | return entry; | |
930 | } else { | |
931 | struct marker_probe_closure *closure; | |
932 | closure = entry->multi; | |
933 | for (i = 0; closure[i].func; i++) { | |
934 | if (closure[i].func == probe && | |
935 | closure[i].probe_private | |
936 | == probe_private) | |
937 | return entry; | |
938 | } | |
939 | } | |
940 | } | |
941 | } | |
942 | return NULL; | |
943 | } | |
944 | ||
945 | /** | |
946 | * marker_probe_unregister_private_data - Disconnect a probe from a marker | |
947 | * @probe: probe function | |
948 | * @probe_private: probe private data | |
949 | * | |
950 | * Unregister a probe by providing the registered private data. | |
951 | * Only removes the first marker found in hash table. | |
952 | * Return 0 on success or error value. | |
953 | * We do not need to call a synchronize_sched to make sure the probes have | |
954 | * finished running before doing a module unload, because the module unload | |
955 | * itself uses stop_machine(), which insures that every preempt disabled section | |
956 | * have finished. | |
957 | */ | |
958 | int marker_probe_unregister_private_data(marker_probe_func *probe, | |
959 | void *probe_private) | |
960 | { | |
961 | struct marker_entry *entry; | |
962 | int ret = 0; | |
963 | struct marker_probe_closure *old; | |
964 | const char *channel = NULL, *name = NULL; | |
965 | ||
966 | mutex_lock(&markers_mutex); | |
967 | entry = get_marker_from_private_data(probe, probe_private); | |
968 | if (!entry) { | |
969 | ret = -ENOENT; | |
970 | goto end; | |
971 | } | |
6cb88bc0 PMF |
972 | //ust// if (entry->rcu_pending) |
973 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
974 | old = marker_entry_remove_probe(entry, NULL, probe_private); |
975 | channel = kstrdup(entry->channel, GFP_KERNEL); | |
976 | name = kstrdup(entry->name, GFP_KERNEL); | |
977 | mutex_unlock(&markers_mutex); | |
978 | ||
979 | marker_update_probes(); | |
980 | ||
981 | mutex_lock(&markers_mutex); | |
982 | entry = get_marker(channel, name); | |
983 | if (!entry) | |
984 | goto end; | |
6cb88bc0 PMF |
985 | //ust// if (entry->rcu_pending) |
986 | //ust// rcu_barrier_sched(); | |
68c1021b PMF |
987 | entry->oldptr = old; |
988 | entry->rcu_pending = 1; | |
989 | /* write rcu_pending before calling the RCU callback */ | |
990 | smp_wmb(); | |
6cb88bc0 PMF |
991 | //ust// call_rcu_sched(&entry->rcu, free_old_closure); |
992 | synchronize_rcu(); free_old_closure(&entry->rcu); | |
68c1021b PMF |
993 | /* Ignore busy error message */ |
994 | remove_marker(channel, name); | |
995 | end: | |
996 | mutex_unlock(&markers_mutex); | |
997 | kfree(channel); | |
998 | kfree(name); | |
999 | return ret; | |
1000 | } | |
59b161cd | 1001 | //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
68c1021b PMF |
1002 | |
1003 | /** | |
1004 | * marker_get_private_data - Get a marker's probe private data | |
1005 | * @channel: marker channel | |
1006 | * @name: marker name | |
1007 | * @probe: probe to match | |
1008 | * @num: get the nth matching probe's private data | |
1009 | * | |
1010 | * Returns the nth private data pointer (starting from 0) matching, or an | |
1011 | * ERR_PTR. | |
1012 | * Returns the private data pointer, or an ERR_PTR. | |
1013 | * The private data pointer should _only_ be dereferenced if the caller is the | |
1014 | * owner of the data, or its content could vanish. This is mostly used to | |
1015 | * confirm that a caller is the owner of a registered probe. | |
1016 | */ | |
1017 | void *marker_get_private_data(const char *channel, const char *name, | |
1018 | marker_probe_func *probe, int num) | |
1019 | { | |
1020 | struct hlist_head *head; | |
1021 | struct hlist_node *node; | |
1022 | struct marker_entry *e; | |
1023 | size_t channel_len = strlen(channel) + 1; | |
1024 | size_t name_len = strlen(name) + 1; | |
1025 | int i; | |
1026 | u32 hash; | |
1027 | ||
1028 | hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); | |
1029 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
1030 | hlist_for_each_entry(e, node, head, hlist) { | |
1031 | if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { | |
1032 | if (!e->ptype) { | |
1033 | if (num == 0 && e->single.func == probe) | |
1034 | return e->single.probe_private; | |
1035 | } else { | |
1036 | struct marker_probe_closure *closure; | |
1037 | int match = 0; | |
1038 | closure = e->multi; | |
1039 | for (i = 0; closure[i].func; i++) { | |
1040 | if (closure[i].func != probe) | |
1041 | continue; | |
1042 | if (match++ == num) | |
1043 | return closure[i].probe_private; | |
1044 | } | |
1045 | } | |
1046 | break; | |
1047 | } | |
1048 | } | |
1049 | return ERR_PTR(-ENOENT); | |
1050 | } | |
59b161cd | 1051 | //ust// EXPORT_SYMBOL_GPL(marker_get_private_data); |
68c1021b PMF |
1052 | |
1053 | /** | |
1054 | * markers_compact_event_ids - Compact markers event IDs and reassign channels | |
1055 | * | |
1056 | * Called when no channel users are active by the channel infrastructure. | |
1057 | * Called with lock_markers() and channel mutex held. | |
1058 | */ | |
59b161cd PMF |
1059 | //ust// void markers_compact_event_ids(void) |
1060 | //ust// { | |
1061 | //ust// struct marker_entry *entry; | |
1062 | //ust// unsigned int i; | |
1063 | //ust// struct hlist_head *head; | |
1064 | //ust// struct hlist_node *node; | |
1065 | //ust// int ret; | |
1066 | //ust// | |
1067 | //ust// for (i = 0; i < MARKER_TABLE_SIZE; i++) { | |
1068 | //ust// head = &marker_table[i]; | |
1069 | //ust// hlist_for_each_entry(entry, node, head, hlist) { | |
1070 | //ust// ret = ltt_channels_get_index_from_name(entry->channel); | |
1071 | //ust// WARN_ON(ret < 0); | |
1072 | //ust// entry->channel_id = ret; | |
1073 | //ust// ret = _ltt_channels_get_event_id(entry->channel, | |
1074 | //ust// entry->name); | |
1075 | //ust// WARN_ON(ret < 0); | |
1076 | //ust// entry->event_id = ret; | |
1077 | //ust// } | |
1078 | //ust// } | |
1079 | //ust// } | |
68c1021b | 1080 | |
9c67dc50 | 1081 | //ust//#ifdef CONFIG_MODULES |
68c1021b | 1082 | |
772030fe PMF |
1083 | /* |
1084 | * Returns 0 if current not found. | |
1085 | * Returns 1 if current found. | |
1086 | */ | |
1087 | int lib_get_iter_markers(struct marker_iter *iter) | |
1088 | { | |
1089 | struct lib *iter_lib; | |
1090 | int found = 0; | |
1091 | ||
1092 | //ust// mutex_lock(&module_mutex); | |
1093 | list_for_each_entry(iter_lib, &libs, list) { | |
1094 | if (iter_lib < iter->lib) | |
1095 | continue; | |
1096 | else if (iter_lib > iter->lib) | |
1097 | iter->marker = NULL; | |
1098 | found = marker_get_iter_range(&iter->marker, | |
1099 | iter_lib->markers_start, | |
1100 | iter_lib->markers_start + iter_lib->markers_count); | |
1101 | if (found) { | |
1102 | iter->lib = iter_lib; | |
1103 | break; | |
1104 | } | |
1105 | } | |
1106 | //ust// mutex_unlock(&module_mutex); | |
1107 | return found; | |
1108 | } | |
1109 | ||
68c1021b PMF |
1110 | /** |
1111 | * marker_get_iter_range - Get a next marker iterator given a range. | |
1112 | * @marker: current markers (in), next marker (out) | |
1113 | * @begin: beginning of the range | |
1114 | * @end: end of the range | |
1115 | * | |
1116 | * Returns whether a next marker has been found (1) or not (0). | |
1117 | * Will return the first marker in the range if the input marker is NULL. | |
1118 | */ | |
1119 | int marker_get_iter_range(struct marker **marker, struct marker *begin, | |
1120 | struct marker *end) | |
1121 | { | |
1122 | if (!*marker && begin != end) { | |
1123 | *marker = begin; | |
1124 | return 1; | |
1125 | } | |
1126 | if (*marker >= begin && *marker < end) | |
1127 | return 1; | |
1128 | return 0; | |
1129 | } | |
59b161cd | 1130 | //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range); |
68c1021b PMF |
1131 | |
1132 | static void marker_get_iter(struct marker_iter *iter) | |
1133 | { | |
1134 | int found = 0; | |
1135 | ||
1136 | /* Core kernel markers */ | |
98963de4 | 1137 | if (!iter->lib) { |
c463904d | 1138 | /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */ |
68c1021b PMF |
1139 | found = marker_get_iter_range(&iter->marker, |
1140 | __start___markers, __stop___markers); | |
1141 | if (found) | |
1142 | goto end; | |
1143 | } | |
1144 | /* Markers in modules. */ | |
98963de4 | 1145 | found = lib_get_iter_markers(iter); |
68c1021b PMF |
1146 | end: |
1147 | if (!found) | |
1148 | marker_iter_reset(iter); | |
1149 | } | |
1150 | ||
1151 | void marker_iter_start(struct marker_iter *iter) | |
1152 | { | |
1153 | marker_get_iter(iter); | |
1154 | } | |
59b161cd | 1155 | //ust// EXPORT_SYMBOL_GPL(marker_iter_start); |
68c1021b PMF |
1156 | |
1157 | void marker_iter_next(struct marker_iter *iter) | |
1158 | { | |
1159 | iter->marker++; | |
1160 | /* | |
1161 | * iter->marker may be invalid because we blindly incremented it. | |
1162 | * Make sure it is valid by marshalling on the markers, getting the | |
1163 | * markers from following modules if necessary. | |
1164 | */ | |
1165 | marker_get_iter(iter); | |
1166 | } | |
59b161cd | 1167 | //ust// EXPORT_SYMBOL_GPL(marker_iter_next); |
68c1021b PMF |
1168 | |
1169 | void marker_iter_stop(struct marker_iter *iter) | |
1170 | { | |
1171 | } | |
59b161cd | 1172 | //ust// EXPORT_SYMBOL_GPL(marker_iter_stop); |
68c1021b PMF |
1173 | |
1174 | void marker_iter_reset(struct marker_iter *iter) | |
1175 | { | |
98963de4 | 1176 | iter->lib = NULL; |
68c1021b PMF |
1177 | iter->marker = NULL; |
1178 | } | |
59b161cd | 1179 | //ust// EXPORT_SYMBOL_GPL(marker_iter_reset); |
68c1021b PMF |
1180 | |
1181 | #ifdef CONFIG_MARKERS_USERSPACE | |
1182 | /* | |
1183 | * must be called with current->user_markers_mutex held | |
1184 | */ | |
1185 | static void free_user_marker(char __user *state, struct hlist_head *head) | |
1186 | { | |
1187 | struct user_marker *umark; | |
1188 | struct hlist_node *pos, *n; | |
1189 | ||
1190 | hlist_for_each_entry_safe(umark, pos, n, head, hlist) { | |
1191 | if (umark->state == state) { | |
1192 | hlist_del(&umark->hlist); | |
1193 | kfree(umark); | |
1194 | } | |
1195 | } | |
1196 | } | |
1197 | ||
59b161cd PMF |
1198 | //ust// asmlinkage long sys_marker(char __user *name, char __user *format, |
1199 | //ust// char __user *state, int reg) | |
1200 | //ust// { | |
1201 | //ust// struct user_marker *umark; | |
1202 | //ust// long len; | |
1203 | //ust// struct marker_entry *entry; | |
1204 | //ust// int ret = 0; | |
1205 | //ust// | |
1206 | //ust// printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n", | |
1207 | //ust// current->comm, reg ? "registers" : "unregisters", | |
1208 | //ust// name, state); | |
1209 | //ust// if (reg) { | |
1210 | //ust// umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL); | |
1211 | //ust// umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0'; | |
1212 | //ust// umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0'; | |
1213 | //ust// umark->state = state; | |
1214 | //ust// len = strncpy_from_user(umark->name, name, | |
1215 | //ust// MAX_USER_MARKER_NAME_LEN - 1); | |
1216 | //ust// if (len < 0) { | |
1217 | //ust// ret = -EFAULT; | |
1218 | //ust// goto error; | |
1219 | //ust// } | |
1220 | //ust// len = strncpy_from_user(umark->format, format, | |
1221 | //ust// MAX_USER_MARKER_FORMAT_LEN - 1); | |
1222 | //ust// if (len < 0) { | |
1223 | //ust// ret = -EFAULT; | |
1224 | //ust// goto error; | |
1225 | //ust// } | |
1226 | //ust// printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name, | |
1227 | //ust// umark->format); | |
1228 | //ust// mutex_lock(&markers_mutex); | |
1229 | //ust// entry = get_marker("userspace", umark->name); | |
1230 | //ust// if (entry) { | |
1231 | //ust// if (entry->format && | |
1232 | //ust// strcmp(entry->format, umark->format) != 0) { | |
1233 | //ust// printk(" error, wrong format in process %s", | |
1234 | //ust// current->comm); | |
1235 | //ust// ret = -EPERM; | |
1236 | //ust// goto error_unlock; | |
1237 | //ust// } | |
1238 | //ust// printk(" %s", !!entry->refcount | |
1239 | //ust// ? "enabled" : "disabled"); | |
1240 | //ust// if (put_user(!!entry->refcount, state)) { | |
1241 | //ust// ret = -EFAULT; | |
1242 | //ust// goto error_unlock; | |
1243 | //ust// } | |
1244 | //ust// printk("\n"); | |
1245 | //ust// } else { | |
1246 | //ust// printk(" disabled\n"); | |
1247 | //ust// if (put_user(0, umark->state)) { | |
1248 | //ust// printk(KERN_WARNING | |
1249 | //ust// "Marker in %s caused a fault\n", | |
1250 | //ust// current->comm); | |
1251 | //ust// goto error_unlock; | |
1252 | //ust// } | |
1253 | //ust// } | |
1254 | //ust// mutex_lock(¤t->group_leader->user_markers_mutex); | |
1255 | //ust// hlist_add_head(&umark->hlist, | |
1256 | //ust// ¤t->group_leader->user_markers); | |
1257 | //ust// current->group_leader->user_markers_sequence++; | |
1258 | //ust// mutex_unlock(¤t->group_leader->user_markers_mutex); | |
1259 | //ust// mutex_unlock(&markers_mutex); | |
1260 | //ust// } else { | |
1261 | //ust// mutex_lock(¤t->group_leader->user_markers_mutex); | |
1262 | //ust// free_user_marker(state, | |
1263 | //ust// ¤t->group_leader->user_markers); | |
1264 | //ust// current->group_leader->user_markers_sequence++; | |
1265 | //ust// mutex_unlock(¤t->group_leader->user_markers_mutex); | |
1266 | //ust// } | |
1267 | //ust// goto end; | |
1268 | //ust// error_unlock: | |
1269 | //ust// mutex_unlock(&markers_mutex); | |
1270 | //ust// error: | |
1271 | //ust// kfree(umark); | |
1272 | //ust// end: | |
1273 | //ust// return ret; | |
1274 | //ust// } | |
1275 | //ust// | |
1276 | //ust// /* | |
1277 | //ust// * Types : | |
1278 | //ust// * string : 0 | |
1279 | //ust// */ | |
1280 | //ust// asmlinkage long sys_trace(int type, uint16_t id, | |
1281 | //ust// char __user *ubuf) | |
1282 | //ust// { | |
1283 | //ust// long ret = -EPERM; | |
1284 | //ust// char *page; | |
1285 | //ust// int len; | |
1286 | //ust// | |
1287 | //ust// switch (type) { | |
1288 | //ust// case 0: /* String */ | |
1289 | //ust// ret = -ENOMEM; | |
1290 | //ust// page = (char *)__get_free_page(GFP_TEMPORARY); | |
1291 | //ust// if (!page) | |
1292 | //ust// goto string_out; | |
1293 | //ust// len = strncpy_from_user(page, ubuf, PAGE_SIZE); | |
1294 | //ust// if (len < 0) { | |
1295 | //ust// ret = -EFAULT; | |
1296 | //ust// goto string_err; | |
1297 | //ust// } | |
1298 | //ust// trace_mark(userspace, string, "string %s", page); | |
1299 | //ust// string_err: | |
1300 | //ust// free_page((unsigned long) page); | |
1301 | //ust// string_out: | |
1302 | //ust// break; | |
1303 | //ust// default: | |
1304 | //ust// break; | |
1305 | //ust// } | |
1306 | //ust// return ret; | |
1307 | //ust// } | |
1308 | ||
1309 | //ust// static void marker_update_processes(void) | |
1310 | //ust// { | |
1311 | //ust// struct task_struct *g, *t; | |
1312 | //ust// | |
1313 | //ust// /* | |
1314 | //ust// * markers_mutex is taken to protect the p->user_markers read. | |
1315 | //ust// */ | |
1316 | //ust// mutex_lock(&markers_mutex); | |
1317 | //ust// read_lock(&tasklist_lock); | |
1318 | //ust// for_each_process(g) { | |
1319 | //ust// WARN_ON(!thread_group_leader(g)); | |
1320 | //ust// if (hlist_empty(&g->user_markers)) | |
1321 | //ust// continue; | |
1322 | //ust// if (strcmp(g->comm, "testprog") == 0) | |
1323 | //ust// printk(KERN_DEBUG "set update pending for testprog\n"); | |
1324 | //ust// t = g; | |
1325 | //ust// do { | |
1326 | //ust// /* TODO : implement this thread flag in each arch. */ | |
1327 | //ust// set_tsk_thread_flag(t, TIF_MARKER_PENDING); | |
1328 | //ust// } while ((t = next_thread(t)) != g); | |
1329 | //ust// } | |
1330 | //ust// read_unlock(&tasklist_lock); | |
1331 | //ust// mutex_unlock(&markers_mutex); | |
1332 | //ust// } | |
68c1021b PMF |
1333 | |
1334 | /* | |
1335 | * Update current process. | |
1336 | * Note that we have to wait a whole scheduler period before we are sure that | |
1337 | * every running userspace threads have their markers updated. | |
1338 | * (synchronize_sched() can be used to insure this). | |
1339 | */ | |
1340 | void marker_update_process(void) | |
1341 | { | |
1342 | struct user_marker *umark; | |
1343 | struct hlist_node *pos; | |
1344 | struct marker_entry *entry; | |
1345 | ||
1346 | mutex_lock(&markers_mutex); | |
1347 | mutex_lock(¤t->group_leader->user_markers_mutex); | |
1348 | if (strcmp(current->comm, "testprog") == 0) | |
1349 | printk(KERN_DEBUG "do update pending for testprog\n"); | |
1350 | hlist_for_each_entry(umark, pos, | |
1351 | ¤t->group_leader->user_markers, hlist) { | |
1352 | printk(KERN_DEBUG "Updating marker %s in %s\n", | |
1353 | umark->name, current->comm); | |
1354 | entry = get_marker("userspace", umark->name); | |
1355 | if (entry) { | |
1356 | if (entry->format && | |
1357 | strcmp(entry->format, umark->format) != 0) { | |
1358 | printk(KERN_WARNING | |
1359 | " error, wrong format in process %s\n", | |
1360 | current->comm); | |
1361 | break; | |
1362 | } | |
1363 | if (put_user(!!entry->refcount, umark->state)) { | |
1364 | printk(KERN_WARNING | |
1365 | "Marker in %s caused a fault\n", | |
1366 | current->comm); | |
1367 | break; | |
1368 | } | |
1369 | } else { | |
1370 | if (put_user(0, umark->state)) { | |
1371 | printk(KERN_WARNING | |
1372 | "Marker in %s caused a fault\n", | |
1373 | current->comm); | |
1374 | break; | |
1375 | } | |
1376 | } | |
1377 | } | |
1378 | clear_thread_flag(TIF_MARKER_PENDING); | |
1379 | mutex_unlock(¤t->group_leader->user_markers_mutex); | |
1380 | mutex_unlock(&markers_mutex); | |
1381 | } | |
1382 | ||
1383 | /* | |
1384 | * Called at process exit and upon do_execve(). | |
1385 | * We assume that when the leader exits, no more references can be done to the | |
1386 | * leader structure by the other threads. | |
1387 | */ | |
1388 | void exit_user_markers(struct task_struct *p) | |
1389 | { | |
1390 | struct user_marker *umark; | |
1391 | struct hlist_node *pos, *n; | |
1392 | ||
1393 | if (thread_group_leader(p)) { | |
1394 | mutex_lock(&markers_mutex); | |
1395 | mutex_lock(&p->user_markers_mutex); | |
1396 | hlist_for_each_entry_safe(umark, pos, n, &p->user_markers, | |
1397 | hlist) | |
1398 | kfree(umark); | |
1399 | INIT_HLIST_HEAD(&p->user_markers); | |
1400 | p->user_markers_sequence++; | |
1401 | mutex_unlock(&p->user_markers_mutex); | |
1402 | mutex_unlock(&markers_mutex); | |
1403 | } | |
1404 | } | |
1405 | ||
1406 | int is_marker_enabled(const char *channel, const char *name) | |
1407 | { | |
1408 | struct marker_entry *entry; | |
1409 | ||
1410 | mutex_lock(&markers_mutex); | |
1411 | entry = get_marker(channel, name); | |
1412 | mutex_unlock(&markers_mutex); | |
1413 | ||
1414 | return entry && !!entry->refcount; | |
1415 | } | |
9c67dc50 | 1416 | //ust// #endif |
68c1021b PMF |
1417 | |
1418 | int marker_module_notify(struct notifier_block *self, | |
1419 | unsigned long val, void *data) | |
1420 | { | |
1421 | struct module *mod = data; | |
1422 | ||
1423 | switch (val) { | |
1424 | case MODULE_STATE_COMING: | |
1425 | marker_update_probe_range(mod->markers, | |
1426 | mod->markers + mod->num_markers); | |
1427 | break; | |
1428 | case MODULE_STATE_GOING: | |
1429 | marker_update_probe_range(mod->markers, | |
1430 | mod->markers + mod->num_markers); | |
1431 | break; | |
1432 | } | |
1433 | return 0; | |
1434 | } | |
1435 | ||
1436 | struct notifier_block marker_module_nb = { | |
1437 | .notifier_call = marker_module_notify, | |
1438 | .priority = 0, | |
1439 | }; | |
1440 | ||
59b161cd PMF |
1441 | //ust// static int init_markers(void) |
1442 | //ust// { | |
1443 | //ust// return register_module_notifier(&marker_module_nb); | |
1444 | //ust// } | |
1445 | //ust// __initcall(init_markers); | |
1446 | /* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */ | |
68c1021b PMF |
1447 | |
1448 | #endif /* CONFIG_MODULES */ | |
1449 | ||
9c67dc50 PMF |
1450 | void ltt_dump_marker_state(struct ltt_trace_struct *trace) |
1451 | { | |
48454c95 | 1452 | struct marker_entry *entry; |
9c67dc50 | 1453 | struct ltt_probe_private_data call_data; |
48454c95 PMF |
1454 | struct hlist_head *head; |
1455 | struct hlist_node *node; | |
1456 | unsigned int i; | |
9c67dc50 | 1457 | |
48454c95 | 1458 | mutex_lock(&markers_mutex); |
9c67dc50 PMF |
1459 | call_data.trace = trace; |
1460 | call_data.serializer = NULL; | |
1461 | ||
48454c95 PMF |
1462 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
1463 | head = &marker_table[i]; | |
1464 | hlist_for_each_entry(entry, node, head, hlist) { | |
1465 | __trace_mark(0, metadata, core_marker_id, | |
9c67dc50 | 1466 | &call_data, |
48454c95 PMF |
1467 | "channel %s name %s event_id %hu " |
1468 | "int #1u%zu long #1u%zu pointer #1u%zu " | |
1469 | "size_t #1u%zu alignment #1u%u", | |
1470 | entry->channel, | |
1471 | entry->name, | |
1472 | entry->event_id, | |
1473 | sizeof(int), sizeof(long), | |
1474 | sizeof(void *), sizeof(size_t), | |
1475 | ltt_get_alignment()); | |
1476 | if (entry->format) | |
1477 | __trace_mark(0, metadata, | |
1478 | core_marker_format, | |
1479 | &call_data, | |
1480 | "channel %s name %s format %s", | |
1481 | entry->channel, | |
1482 | entry->name, | |
1483 | entry->format); | |
1484 | } | |
9c67dc50 | 1485 | } |
48454c95 | 1486 | mutex_unlock(&markers_mutex); |
9c67dc50 | 1487 | } |
59b161cd | 1488 | //ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state); |
98963de4 | 1489 | |
20b37a31 PMF |
1490 | static void (*new_marker_cb)(struct marker *) = NULL; |
1491 | ||
1492 | void marker_set_new_marker_cb(void (*cb)(struct marker *)) | |
1493 | { | |
1494 | new_marker_cb = cb; | |
1495 | } | |
1496 | ||
1497 | static void new_markers(struct marker *start, struct marker *end) | |
1498 | { | |
1499 | if(new_marker_cb) { | |
1500 | struct marker *m; | |
1501 | for(m=start; m < end; m++) { | |
1502 | new_marker_cb(m); | |
1503 | } | |
1504 | } | |
1505 | } | |
1506 | ||
3ea1e2fc | 1507 | int marker_register_lib(struct marker *markers_start, struct marker_addr *marker_addr_start, int markers_count) |
98963de4 PMF |
1508 | { |
1509 | struct lib *pl; | |
3ea1e2fc | 1510 | struct marker_addr *addr; |
98963de4 PMF |
1511 | |
1512 | pl = (struct lib *) malloc(sizeof(struct lib)); | |
1513 | ||
1514 | pl->markers_start = markers_start; | |
defa46a7 | 1515 | #ifdef CONFIG_UST_GDB_INTEGRATION |
3ea1e2fc | 1516 | pl->markers_addr_start = marker_addr_start; |
defa46a7 | 1517 | #endif |
98963de4 PMF |
1518 | pl->markers_count = markers_count; |
1519 | ||
3c44233c | 1520 | #ifdef CONFIG_UST_GDB_INTEGRATION |
3ea1e2fc PMF |
1521 | lock_markers(); |
1522 | for(addr = marker_addr_start; addr < marker_addr_start + markers_count; addr++) | |
1523 | addr->marker->location = addr->addr; | |
1524 | unlock_markers(); | |
3c44233c | 1525 | #endif |
3ea1e2fc | 1526 | |
0b5207fa PMF |
1527 | /* FIXME: maybe protect this with its own mutex? */ |
1528 | lock_markers(); | |
98963de4 | 1529 | list_add(&pl->list, &libs); |
0b5207fa | 1530 | unlock_markers(); |
98963de4 | 1531 | |
20b37a31 PMF |
1532 | new_markers(markers_start, markers_start + markers_count); |
1533 | ||
4db647c5 PMF |
1534 | /* FIXME: update just the loaded lib */ |
1535 | lib_update_markers(); | |
1536 | ||
1c184644 | 1537 | DBG("just registered a markers section from %p and having %d markers", markers_start, markers_count); |
98963de4 PMF |
1538 | |
1539 | return 0; | |
1540 | } | |
c463904d | 1541 | |
0b5207fa PMF |
1542 | int marker_unregister_lib(struct marker *markers_start, int markers_count) |
1543 | { | |
1544 | /*FIXME: implement; but before implementing, marker_register_lib must | |
1545 | have appropriate locking. */ | |
1546 | ||
1547 | return 0; | |
1548 | } | |
1549 | ||
4db647c5 PMF |
1550 | static int initialized = 0; |
1551 | ||
1552 | void __attribute__((constructor)) init_markers(void) | |
c463904d | 1553 | { |
4db647c5 | 1554 | if(!initialized) { |
defa46a7 | 1555 | #ifdef CONFIG_UST_GDB_INTEGRATION |
3ea1e2fc | 1556 | marker_register_lib(__start___markers, __start___marker_addr, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker)); |
defa46a7 PMF |
1557 | #else |
1558 | marker_register_lib(__start___markers, NULL, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker)); | |
1559 | #endif | |
3ea1e2fc | 1560 | //DBG("markers_start: %p, markers_stop: %p\n", __start___markers, __stop___markers); |
4db647c5 PMF |
1561 | initialized = 1; |
1562 | } | |
c463904d | 1563 | } |