b6a2181ac19b58ea921c3677ab5d4c6f52b97117
[lttng-ust.git] / libust / relay.c
1 /*
2 * Public API and common code for kernel->userspace relay file support.
3 *
4 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
5 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
6 * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca)
7 *
8 * Moved to kernel/relay.c by Paul Mundt, 2006.
9 * November 2006 - CPU hotplug support by Mathieu Desnoyers
10 * (mathieu.desnoyers@polymtl.ca)
11 *
12 * This file is released under the GPL.
13 */
14 //ust// #include <linux/errno.h>
15 //ust// #include <linux/stddef.h>
16 //ust// #include <linux/slab.h>
17 //ust// #include <linux/module.h>
18 //ust// #include <linux/string.h>
19 //ust// #include <linux/ltt-relay.h>
20 //ust// #include <linux/vmalloc.h>
21 //ust// #include <linux/mm.h>
22 //ust// #include <linux/cpu.h>
23 //ust// #include <linux/splice.h>
24 //ust// #include <linux/bitops.h>
25 #include <ust/kernelcompat.h>
26 #include <sys/mman.h>
27 #include <sys/ipc.h>
28 #include <sys/shm.h>
29 #include <fcntl.h>
30 //#include "list.h"
31 #include "relay.h"
32 #include "channels.h"
33 #include <kcompat/kref.h>
34 #include <ust/tracer.h>
35 #include "tracercore.h"
36 #include "usterr.h"
37
38 /* list of open channels, for cpu hotplug */
39 static DEFINE_MUTEX(relay_channels_mutex);
40 static LIST_HEAD(relay_channels);
41
42
43 static struct dentry *ltt_create_buf_file_callback(struct rchan_buf *buf);
44
45 /**
46 * relay_alloc_buf - allocate a channel buffer
47 * @buf: the buffer struct
48 * @size: total size of the buffer
49 */
50 //ust// static int relay_alloc_buf(struct rchan_buf *buf, size_t *size)
51 //ust//{
52 //ust// unsigned int i, n_pages;
53 //ust// struct buf_page *buf_page, *n;
54 //ust//
55 //ust// *size = PAGE_ALIGN(*size);
56 //ust// n_pages = *size >> PAGE_SHIFT;
57 //ust//
58 //ust// INIT_LIST_HEAD(&buf->pages);
59 //ust//
60 //ust// for (i = 0; i < n_pages; i++) {
61 //ust// buf_page = kmalloc_node(sizeof(*buf_page), GFP_KERNEL,
62 //ust// cpu_to_node(buf->cpu));
63 //ust// if (unlikely(!buf_page))
64 //ust// goto depopulate;
65 //ust// buf_page->page = alloc_pages_node(cpu_to_node(buf->cpu),
66 //ust// GFP_KERNEL | __GFP_ZERO, 0);
67 //ust// if (unlikely(!buf_page->page)) {
68 //ust// kfree(buf_page);
69 //ust// goto depopulate;
70 //ust// }
71 //ust// list_add_tail(&buf_page->list, &buf->pages);
72 //ust// buf_page->offset = (size_t)i << PAGE_SHIFT;
73 //ust// buf_page->buf = buf;
74 //ust// set_page_private(buf_page->page, (unsigned long)buf_page);
75 //ust// if (i == 0) {
76 //ust// buf->wpage = buf_page;
77 //ust// buf->hpage[0] = buf_page;
78 //ust// buf->hpage[1] = buf_page;
79 //ust// buf->rpage = buf_page;
80 //ust// }
81 //ust// }
82 //ust// buf->page_count = n_pages;
83 //ust// return 0;
84 //ust//
85 //ust//depopulate:
86 //ust// list_for_each_entry_safe(buf_page, n, &buf->pages, list) {
87 //ust// list_del_init(&buf_page->list);
88 //ust// __free_page(buf_page->page);
89 //ust// kfree(buf_page);
90 //ust// }
91 //ust// return -ENOMEM;
92 //ust//}
93
94 static int relay_alloc_buf(struct rchan_buf *buf, size_t *size)
95 {
96 //ust// unsigned int n_pages;
97 //ust// struct buf_page *buf_page, *n;
98
99 void *ptr;
100 int result;
101
102 *size = PAGE_ALIGN(*size);
103
104 result = buf->shmid = shmget(getpid(), *size, IPC_CREAT | IPC_EXCL | 0700);
105 if(result == -1 && errno == EINVAL) {
106 ERR("shmget() returned EINVAL; maybe /proc/sys/kernel/shmmax should be increased.");
107 return -1;
108 }
109 else if(result == -1) {
110 PERROR("shmget");
111 return -1;
112 }
113
114 ptr = shmat(buf->shmid, NULL, 0);
115 if(ptr == (void *) -1) {
116 perror("shmat");
117 goto destroy_shmem;
118 }
119
120 /* Already mark the shared memory for destruction. This will occur only
121 * when all users have detached.
122 */
123 result = shmctl(buf->shmid, IPC_RMID, NULL);
124 if(result == -1) {
125 perror("shmctl");
126 return -1;
127 }
128
129 buf->buf_data = ptr;
130 buf->buf_size = *size;
131
132 return 0;
133
134 destroy_shmem:
135 result = shmctl(buf->shmid, IPC_RMID, NULL);
136 if(result == -1) {
137 perror("shmctl");
138 }
139
140 return -1;
141 }
142
143 /**
144 * relay_create_buf - allocate and initialize a channel buffer
145 * @chan: the relay channel
146 * @cpu: cpu the buffer belongs to
147 *
148 * Returns channel buffer if successful, %NULL otherwise.
149 */
150 static struct rchan_buf *relay_create_buf(struct rchan *chan)
151 {
152 int ret;
153 struct rchan_buf *buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);
154 if (!buf)
155 return NULL;
156
157 // buf->cpu = cpu;
158 ret = relay_alloc_buf(buf, &chan->alloc_size);
159 if (ret)
160 goto free_buf;
161
162 buf->chan = chan;
163 kref_get(&buf->chan->kref);
164 return buf;
165
166 free_buf:
167 kfree(buf);
168 return NULL;
169 }
170
171 /**
172 * relay_destroy_channel - free the channel struct
173 * @kref: target kernel reference that contains the relay channel
174 *
175 * Should only be called from kref_put().
176 */
177 static void relay_destroy_channel(struct kref *kref)
178 {
179 struct rchan *chan = container_of(kref, struct rchan, kref);
180 kfree(chan);
181 }
182
183 /**
184 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
185 * @buf: the buffer struct
186 */
187 static void relay_destroy_buf(struct rchan_buf *buf)
188 {
189 struct rchan *chan = buf->chan;
190 //ust// struct buf_page *buf_page;
191 int result;
192
193 result = munmap(buf->buf_data, buf->buf_size);
194 if(result == -1) {
195 PERROR("munmap");
196 }
197
198 //ust// chan->buf[buf->cpu] = NULL;
199 kfree(buf);
200 kref_put(&chan->kref, relay_destroy_channel);
201 }
202
203 /**
204 * relay_remove_buf - remove a channel buffer
205 * @kref: target kernel reference that contains the relay buffer
206 *
207 * Removes the file from the fileystem, which also frees the
208 * rchan_buf_struct and the channel buffer. Should only be called from
209 * kref_put().
210 */
211 static void relay_remove_buf(struct kref *kref)
212 {
213 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);
214 //ust// buf->chan->cb->remove_buf_file(buf);
215 relay_destroy_buf(buf);
216 }
217
218 /*
219 * High-level relay kernel API and associated functions.
220 */
221
222 /*
223 * rchan_callback implementations defining default channel behavior. Used
224 * in place of corresponding NULL values in client callback struct.
225 */
226
227 /*
228 * create_buf_file_create() default callback. Does nothing.
229 */
230 //ust// static struct dentry *create_buf_file_default_callback(const char *filename,
231 //ust// struct dentry *parent,
232 //ust// int mode,
233 //ust// struct rchan_buf *buf)
234 //ust// {
235 //ust// return NULL;
236 //ust// }
237
238 //ust// /*
239 //ust// * remove_buf_file() default callback. Does nothing.
240 //ust// */
241 //ust// static int remove_buf_file_default_callback(struct dentry *dentry)
242 //ust// {
243 //ust// return -EINVAL;
244 //ust// }
245
246 /**
247 * wakeup_readers - wake up readers waiting on a channel
248 * @data: contains the channel buffer
249 *
250 * This is the timer function used to defer reader waking.
251 */
252 //ust// static void wakeup_readers(unsigned long data)
253 //ust// {
254 //ust// struct rchan_buf *buf = (struct rchan_buf *)data;
255 //ust// wake_up_interruptible(&buf->read_wait);
256 //ust// }
257
258 /**
259 * __relay_reset - reset a channel buffer
260 * @buf: the channel buffer
261 * @init: 1 if this is a first-time initialization
262 *
263 * See relay_reset() for description of effect.
264 */
265 static void __relay_reset(struct rchan_buf *buf, unsigned int init)
266 {
267 if (init) {
268 //ust// init_waitqueue_head(&buf->read_wait);
269 kref_init(&buf->kref);
270 //ust// setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf);
271 } //ust// else
272 //ust// del_timer_sync(&buf->timer);
273
274 buf->finalized = 0;
275 }
276
277 /*
278 * relay_open_buf - create a new relay channel buffer
279 *
280 * used by relay_open() and CPU hotplug.
281 */
282 static struct rchan_buf *relay_open_buf(struct rchan *chan)
283 {
284 struct rchan_buf *buf = NULL;
285 //ust// struct dentry *dentry;
286 //ust// char *tmpname;
287
288 //ust// tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
289 //ust// if (!tmpname)
290 //ust// goto end;
291 //ust// snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
292
293 buf = relay_create_buf(chan);
294 if (!buf)
295 goto free_name;
296
297 __relay_reset(buf, 1);
298
299 /* Create file in fs */
300 //ust// dentry = chan->cb->create_buf_file(tmpname, chan->parent, S_IRUSR,
301 //ust// buf);
302
303 ltt_create_buf_file_callback(buf); // ust //
304
305 //ust// if (!dentry)
306 //ust// goto free_buf;
307 //ust//
308 //ust// buf->dentry = dentry;
309
310 goto free_name;
311
312 //ust//free_buf:
313 relay_destroy_buf(buf);
314 buf = NULL;
315 free_name:
316 //ust// kfree(tmpname);
317 //ust//end:
318 return buf;
319 }
320
321 /**
322 * relay_close_buf - close a channel buffer
323 * @buf: channel buffer
324 *
325 * Marks the buffer finalized and restores the default callbacks.
326 * The channel buffer and channel buffer data structure are then freed
327 * automatically when the last reference is given up.
328 */
329 static void relay_close_buf(struct rchan_buf *buf)
330 {
331 //ust// del_timer_sync(&buf->timer);
332 kref_put(&buf->kref, relay_remove_buf);
333 }
334
335 //ust// static void setup_callbacks(struct rchan *chan,
336 //ust// struct rchan_callbacks *cb)
337 //ust// {
338 //ust// if (!cb) {
339 //ust// chan->cb = &default_channel_callbacks;
340 //ust// return;
341 //ust// }
342 //ust//
343 //ust// if (!cb->create_buf_file)
344 //ust// cb->create_buf_file = create_buf_file_default_callback;
345 //ust// if (!cb->remove_buf_file)
346 //ust// cb->remove_buf_file = remove_buf_file_default_callback;
347 //ust// chan->cb = cb;
348 //ust// }
349
350 /**
351 * relay_hotcpu_callback - CPU hotplug callback
352 * @nb: notifier block
353 * @action: hotplug action to take
354 * @hcpu: CPU number
355 *
356 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
357 */
358 //ust// static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb,
359 //ust// unsigned long action,
360 //ust// void *hcpu)
361 //ust// {
362 //ust// unsigned int hotcpu = (unsigned long)hcpu;
363 //ust// struct rchan *chan;
364 //ust//
365 //ust// switch (action) {
366 //ust// case CPU_UP_PREPARE:
367 //ust// case CPU_UP_PREPARE_FROZEN:
368 //ust// mutex_lock(&relay_channels_mutex);
369 //ust// list_for_each_entry(chan, &relay_channels, list) {
370 //ust// if (chan->buf[hotcpu])
371 //ust// continue;
372 //ust// chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);
373 //ust// if (!chan->buf[hotcpu]) {
374 //ust// printk(KERN_ERR
375 //ust// "relay_hotcpu_callback: cpu %d buffer "
376 //ust// "creation failed\n", hotcpu);
377 //ust// mutex_unlock(&relay_channels_mutex);
378 //ust// return NOTIFY_BAD;
379 //ust// }
380 //ust// }
381 //ust// mutex_unlock(&relay_channels_mutex);
382 //ust// break;
383 //ust// case CPU_DEAD:
384 //ust// case CPU_DEAD_FROZEN:
385 //ust// /* No need to flush the cpu : will be flushed upon
386 //ust// * final relay_flush() call. */
387 //ust// break;
388 //ust// }
389 //ust// return NOTIFY_OK;
390 //ust// }
391
392 /**
393 * ltt_relay_open - create a new relay channel
394 * @base_filename: base name of files to create
395 * @parent: dentry of parent directory, %NULL for root directory
396 * @subbuf_size: size of sub-buffers
397 * @n_subbufs: number of sub-buffers
398 * @cb: client callback functions
399 * @private_data: user-defined data
400 *
401 * Returns channel pointer if successful, %NULL otherwise.
402 *
403 * Creates a channel buffer for each cpu using the sizes and
404 * attributes specified. The created channel buffer files
405 * will be named base_filename0...base_filenameN-1. File
406 * permissions will be %S_IRUSR.
407 */
408 struct rchan *ltt_relay_open(const char *base_filename,
409 struct dentry *parent,
410 size_t subbuf_size,
411 size_t n_subbufs,
412 void *private_data)
413 {
414 //ust// unsigned int i;
415 struct rchan *chan;
416 //ust// if (!base_filename)
417 //ust// return NULL;
418
419 if (!(subbuf_size && n_subbufs))
420 return NULL;
421
422 chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);
423 if (!chan)
424 return NULL;
425
426 chan->version = LTT_RELAY_CHANNEL_VERSION;
427 chan->n_subbufs = n_subbufs;
428 chan->subbuf_size = subbuf_size;
429 chan->subbuf_size_order = get_count_order(subbuf_size);
430 chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs);
431 chan->parent = parent;
432 chan->private_data = private_data;
433 //ust// strlcpy(chan->base_filename, base_filename, NAME_MAX);
434 //ust// setup_callbacks(chan, cb);
435 kref_init(&chan->kref);
436
437 mutex_lock(&relay_channels_mutex);
438 //ust// for_each_online_cpu(i) {
439 chan->buf = relay_open_buf(chan);
440 if (!chan->buf)
441 goto error;
442 //ust// }
443 list_add(&chan->list, &relay_channels);
444 mutex_unlock(&relay_channels_mutex);
445
446 return chan;
447
448 //ust//free_bufs:
449 //ust// for_each_possible_cpu(i) {
450 //ust// if (!chan->buf[i])
451 //ust// break;
452 //ust// relay_close_buf(chan->buf[i]);
453 //ust// }
454
455 error:
456 kref_put(&chan->kref, relay_destroy_channel);
457 mutex_unlock(&relay_channels_mutex);
458 return NULL;
459 }
460 //ust// EXPORT_SYMBOL_GPL(ltt_relay_open);
461
462 /**
463 * ltt_relay_close - close the channel
464 * @chan: the channel
465 *
466 * Closes all channel buffers and frees the channel.
467 */
468 void ltt_relay_close(struct rchan *chan)
469 {
470 //ust// unsigned int i;
471
472 if (!chan)
473 return;
474
475 mutex_lock(&relay_channels_mutex);
476 //ust// for_each_possible_cpu(i)
477 if (chan->buf)
478 relay_close_buf(chan->buf);
479
480 list_del(&chan->list);
481 kref_put(&chan->kref, relay_destroy_channel);
482 mutex_unlock(&relay_channels_mutex);
483 }
484 //ust// EXPORT_SYMBOL_GPL(ltt_relay_close);
485
486 /*
487 * Start iteration at the previous element. Skip the real list head.
488 */
489 //ust// struct buf_page *ltt_relay_find_prev_page(struct rchan_buf *buf,
490 //ust// struct buf_page *page, size_t offset, ssize_t diff_offset)
491 //ust// {
492 //ust// struct buf_page *iter;
493 //ust// size_t orig_iter_off;
494 //ust// unsigned int i = 0;
495 //ust//
496 //ust// orig_iter_off = page->offset;
497 //ust// list_for_each_entry_reverse(iter, &page->list, list) {
498 //ust// /*
499 //ust// * Skip the real list head.
500 //ust// */
501 //ust// if (&iter->list == &buf->pages)
502 //ust// continue;
503 //ust// i++;
504 //ust// if (offset >= iter->offset
505 //ust// && offset < iter->offset + PAGE_SIZE) {
506 //ust// #ifdef CONFIG_LTT_RELAY_CHECK_RANDOM_ACCESS
507 //ust// if (i > 1) {
508 //ust// printk(KERN_WARNING
509 //ust// "Backward random access detected in "
510 //ust// "ltt_relay. Iterations %u, "
511 //ust// "offset %zu, orig iter->off %zu, "
512 //ust// "iter->off %zu diff_offset %zd.\n", i,
513 //ust// offset, orig_iter_off, iter->offset,
514 //ust// diff_offset);
515 //ust// WARN_ON(1);
516 //ust// }
517 //ust// #endif
518 //ust// return iter;
519 //ust// }
520 //ust// }
521 //ust// WARN_ON(1);
522 //ust// return NULL;
523 //ust// }
524 //ust// EXPORT_SYMBOL_GPL(ltt_relay_find_prev_page);
525
526 /*
527 * Start iteration at the next element. Skip the real list head.
528 */
529 //ust// struct buf_page *ltt_relay_find_next_page(struct rchan_buf *buf,
530 //ust// struct buf_page *page, size_t offset, ssize_t diff_offset)
531 //ust// {
532 //ust// struct buf_page *iter;
533 //ust// unsigned int i = 0;
534 //ust// size_t orig_iter_off;
535 //ust//
536 //ust// orig_iter_off = page->offset;
537 //ust// list_for_each_entry(iter, &page->list, list) {
538 //ust// /*
539 //ust// * Skip the real list head.
540 //ust// */
541 //ust// if (&iter->list == &buf->pages)
542 //ust// continue;
543 //ust// i++;
544 //ust// if (offset >= iter->offset
545 //ust// && offset < iter->offset + PAGE_SIZE) {
546 //ust// #ifdef CONFIG_LTT_RELAY_CHECK_RANDOM_ACCESS
547 //ust// if (i > 1) {
548 //ust// printk(KERN_WARNING
549 //ust// "Forward random access detected in "
550 //ust// "ltt_relay. Iterations %u, "
551 //ust// "offset %zu, orig iter->off %zu, "
552 //ust// "iter->off %zu diff_offset %zd.\n", i,
553 //ust// offset, orig_iter_off, iter->offset,
554 //ust// diff_offset);
555 //ust// WARN_ON(1);
556 //ust// }
557 //ust// #endif
558 //ust// return iter;
559 //ust// }
560 //ust// }
561 //ust// WARN_ON(1);
562 //ust// return NULL;
563 //ust// }
564 //ust// EXPORT_SYMBOL_GPL(ltt_relay_find_next_page);
565
566 /**
567 * ltt_relay_write - write data to a ltt_relay buffer.
568 * @buf : buffer
569 * @offset : offset within the buffer
570 * @src : source address
571 * @len : length to write
572 * @page : cached buffer page
573 * @pagecpy : page size copied so far
574 */
575 void _ltt_relay_write(struct rchan_buf *buf, size_t offset,
576 const void *src, size_t len, ssize_t cpy)
577 {
578 do {
579 len -= cpy;
580 src += cpy;
581 offset += cpy;
582 /*
583 * Underlying layer should never ask for writes across
584 * subbuffers.
585 */
586 WARN_ON(offset >= buf->buf_size);
587
588 cpy = min_t(size_t, len, buf->buf_size - offset);
589 ltt_relay_do_copy(buf->buf_data + offset, src, cpy);
590 } while (unlikely(len != cpy));
591 }
592 //ust// EXPORT_SYMBOL_GPL(_ltt_relay_write);
593
594 /**
595 * ltt_relay_read - read data from ltt_relay_buffer.
596 * @buf : buffer
597 * @offset : offset within the buffer
598 * @dest : destination address
599 * @len : length to write
600 */
601 //ust// int ltt_relay_read(struct rchan_buf *buf, size_t offset,
602 //ust// void *dest, size_t len)
603 //ust// {
604 //ust// struct buf_page *page;
605 //ust// ssize_t pagecpy, orig_len;
606 //ust//
607 //ust// orig_len = len;
608 //ust// offset &= buf->chan->alloc_size - 1;
609 //ust// page = buf->rpage;
610 //ust// if (unlikely(!len))
611 //ust// return 0;
612 //ust// for (;;) {
613 //ust// page = ltt_relay_cache_page(buf, &buf->rpage, page, offset);
614 //ust// pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK));
615 //ust// memcpy(dest, page_address(page->page) + (offset & ~PAGE_MASK),
616 //ust// pagecpy);
617 //ust// len -= pagecpy;
618 //ust// if (likely(!len))
619 //ust// break;
620 //ust// dest += pagecpy;
621 //ust// offset += pagecpy;
622 //ust// /*
623 //ust// * Underlying layer should never ask for reads across
624 //ust// * subbuffers.
625 //ust// */
626 //ust// WARN_ON(offset >= buf->chan->alloc_size);
627 //ust// }
628 //ust// return orig_len;
629 //ust// }
630 //ust// EXPORT_SYMBOL_GPL(ltt_relay_read);
631
632 /**
633 * ltt_relay_read_get_page - Get a whole page to read from
634 * @buf : buffer
635 * @offset : offset within the buffer
636 */
637 //ust// struct buf_page *ltt_relay_read_get_page(struct rchan_buf *buf, size_t offset)
638 //ust// {
639 //ust// struct buf_page *page;
640
641 //ust// offset &= buf->chan->alloc_size - 1;
642 //ust// page = buf->rpage;
643 //ust// page = ltt_relay_cache_page(buf, &buf->rpage, page, offset);
644 //ust// return page;
645 //ust// }
646 //ust// EXPORT_SYMBOL_GPL(ltt_relay_read_get_page);
647
648 /**
649 * ltt_relay_offset_address - get address of a location within the buffer
650 * @buf : buffer
651 * @offset : offset within the buffer.
652 *
653 * Return the address where a given offset is located.
654 * Should be used to get the current subbuffer header pointer. Given we know
655 * it's never on a page boundary, it's safe to write directly to this address,
656 * as long as the write is never bigger than a page size.
657 */
658 void *ltt_relay_offset_address(struct rchan_buf *buf, size_t offset)
659 {
660 //ust// struct buf_page *page;
661 //ust// unsigned int odd;
662 //ust//
663 //ust// offset &= buf->chan->alloc_size - 1;
664 //ust// odd = !!(offset & buf->chan->subbuf_size);
665 //ust// page = buf->hpage[odd];
666 //ust// if (offset < page->offset || offset >= page->offset + PAGE_SIZE)
667 //ust// buf->hpage[odd] = page = buf->wpage;
668 //ust// page = ltt_relay_cache_page(buf, &buf->hpage[odd], page, offset);
669 //ust// return page_address(page->page) + (offset & ~PAGE_MASK);
670 return ((char *)buf->buf_data)+offset;
671 return NULL;
672 }
673 //ust// EXPORT_SYMBOL_GPL(ltt_relay_offset_address);
674
675 /**
676 * relay_file_open - open file op for relay files
677 * @inode: the inode
678 * @filp: the file
679 *
680 * Increments the channel buffer refcount.
681 */
682 //ust// static int relay_file_open(struct inode *inode, struct file *filp)
683 //ust// {
684 //ust// struct rchan_buf *buf = inode->i_private;
685 //ust// kref_get(&buf->kref);
686 //ust// filp->private_data = buf;
687 //ust//
688 //ust// return nonseekable_open(inode, filp);
689 //ust// }
690
691 /**
692 * relay_file_release - release file op for relay files
693 * @inode: the inode
694 * @filp: the file
695 *
696 * Decrements the channel refcount, as the filesystem is
697 * no longer using it.
698 */
699 //ust// static int relay_file_release(struct inode *inode, struct file *filp)
700 //ust// {
701 //ust// struct rchan_buf *buf = filp->private_data;
702 //ust// kref_put(&buf->kref, relay_remove_buf);
703 //ust//
704 //ust// return 0;
705 //ust// }
706
707 //ust// const struct file_operations ltt_relay_file_operations = {
708 //ust// .open = relay_file_open,
709 //ust// .release = relay_file_release,
710 //ust// };
711 //ust// EXPORT_SYMBOL_GPL(ltt_relay_file_operations);
712
713 //ust// static __init int relay_init(void)
714 //ust// {
715 //ust// hotcpu_notifier(relay_hotcpu_callback, 5);
716 //ust// return 0;
717 //ust// }
718
719 //ust// module_init(relay_init);
720 /*
721 * ltt/ltt-relay.c
722 *
723 * (C) Copyright 2005-2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca)
724 *
725 * LTTng lockless buffer space management (reader/writer).
726 *
727 * Author:
728 * Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca)
729 *
730 * Inspired from LTT :
731 * Karim Yaghmour (karim@opersys.com)
732 * Tom Zanussi (zanussi@us.ibm.com)
733 * Bob Wisniewski (bob@watson.ibm.com)
734 * And from K42 :
735 * Bob Wisniewski (bob@watson.ibm.com)
736 *
737 * Changelog:
738 * 08/10/08, Cleanup.
739 * 19/10/05, Complete lockless mechanism.
740 * 27/05/05, Modular redesign and rewrite.
741 *
742 * Userspace reader semantic :
743 * while (poll fd != POLLHUP) {
744 * - ioctl RELAY_GET_SUBBUF_SIZE
745 * while (1) {
746 * - ioctl GET_SUBBUF
747 * - splice 1 subbuffer worth of data to a pipe
748 * - splice the data from pipe to disk/network
749 * - ioctl PUT_SUBBUF, check error value
750 * if err val < 0, previous subbuffer was corrupted.
751 * }
752 * }
753 */
754
755 //ust// #include <linux/time.h>
756 //ust// #include <linux/ltt-tracer.h>
757 //ust// #include <linux/ltt-relay.h>
758 //ust// #include <linux/module.h>
759 //ust// #include <linux/string.h>
760 //ust// #include <linux/slab.h>
761 //ust// #include <linux/init.h>
762 //ust// #include <linux/rcupdate.h>
763 //ust// #include <linux/sched.h>
764 //ust// #include <linux/bitops.h>
765 //ust// #include <linux/fs.h>
766 //ust// #include <linux/smp_lock.h>
767 //ust// #include <linux/debugfs.h>
768 //ust// #include <linux/stat.h>
769 //ust// #include <linux/cpu.h>
770 //ust// #include <linux/pipe_fs_i.h>
771 //ust// #include <linux/splice.h>
772 //ust// #include <asm/atomic.h>
773 //ust// #include <asm/local.h>
774
775 #if 0
776 #define printk_dbg(fmt, args...) printk(fmt, args)
777 #else
778 #define printk_dbg(fmt, args...)
779 #endif
780
781 /*
782 * Last TSC comparison functions. Check if the current TSC overflows
783 * LTT_TSC_BITS bits from the last TSC read. Reads and writes last_tsc
784 * atomically.
785 */
786
787 #if (BITS_PER_LONG == 32)
788 static inline void save_last_tsc(struct ltt_channel_buf_struct *ltt_buf,
789 u64 tsc)
790 {
791 ltt_buf->last_tsc = (unsigned long)(tsc >> LTT_TSC_BITS);
792 }
793
794 static inline int last_tsc_overflow(struct ltt_channel_buf_struct *ltt_buf,
795 u64 tsc)
796 {
797 unsigned long tsc_shifted = (unsigned long)(tsc >> LTT_TSC_BITS);
798
799 if (unlikely((tsc_shifted - ltt_buf->last_tsc)))
800 return 1;
801 else
802 return 0;
803 }
804 #else
805 static inline void save_last_tsc(struct ltt_channel_buf_struct *ltt_buf,
806 u64 tsc)
807 {
808 ltt_buf->last_tsc = (unsigned long)tsc;
809 }
810
811 static inline int last_tsc_overflow(struct ltt_channel_buf_struct *ltt_buf,
812 u64 tsc)
813 {
814 if (unlikely((tsc - ltt_buf->last_tsc) >> LTT_TSC_BITS))
815 return 1;
816 else
817 return 0;
818 }
819 #endif
820
821 //ust// static struct file_operations ltt_file_operations;
822
823 /*
824 * A switch is done during tracing or as a final flush after tracing (so it
825 * won't write in the new sub-buffer).
826 */
827 enum force_switch_mode { FORCE_ACTIVE, FORCE_FLUSH };
828
829 static int ltt_relay_create_buffer(struct ltt_trace_struct *trace,
830 struct ltt_channel_struct *ltt_chan,
831 struct rchan_buf *buf,
832 unsigned int n_subbufs);
833
834 static void ltt_relay_destroy_buffer(struct ltt_channel_struct *ltt_chan);
835
836 static void ltt_force_switch(struct rchan_buf *buf,
837 enum force_switch_mode mode);
838
839 /*
840 * Trace callbacks
841 */
842 static void ltt_buffer_begin_callback(struct rchan_buf *buf,
843 u64 tsc, unsigned int subbuf_idx)
844 {
845 struct ltt_channel_struct *channel =
846 (struct ltt_channel_struct *)buf->chan->private_data;
847 struct ltt_subbuffer_header *header =
848 (struct ltt_subbuffer_header *)
849 ltt_relay_offset_address(buf,
850 subbuf_idx * buf->chan->subbuf_size);
851
852 header->cycle_count_begin = tsc;
853 header->lost_size = 0xFFFFFFFF; /* for debugging */
854 header->buf_size = buf->chan->subbuf_size;
855 ltt_write_trace_header(channel->trace, header);
856 }
857
858 /*
859 * offset is assumed to never be 0 here : never deliver a completely empty
860 * subbuffer. The lost size is between 0 and subbuf_size-1.
861 */
862 static notrace void ltt_buffer_end_callback(struct rchan_buf *buf,
863 u64 tsc, unsigned int offset, unsigned int subbuf_idx)
864 {
865 struct ltt_channel_struct *channel =
866 (struct ltt_channel_struct *)buf->chan->private_data;
867 struct ltt_channel_buf_struct *ltt_buf = channel->buf;
868 struct ltt_subbuffer_header *header =
869 (struct ltt_subbuffer_header *)
870 ltt_relay_offset_address(buf,
871 subbuf_idx * buf->chan->subbuf_size);
872
873 header->lost_size = SUBBUF_OFFSET((buf->chan->subbuf_size - offset),
874 buf->chan);
875 header->cycle_count_end = tsc;
876 header->events_lost = local_read(&ltt_buf->events_lost);
877 header->subbuf_corrupt = local_read(&ltt_buf->corrupted_subbuffers);
878
879 }
880
881 void (*wake_consumer)(void *, int) = NULL;
882
883 void relay_set_wake_consumer(void (*wake)(void *, int))
884 {
885 wake_consumer = wake;
886 }
887
888 void relay_wake_consumer(void *arg, int finished)
889 {
890 if(wake_consumer)
891 wake_consumer(arg, finished);
892 }
893
894 static notrace void ltt_deliver(struct rchan_buf *buf, unsigned int subbuf_idx,
895 long commit_count)
896 {
897 struct ltt_channel_struct *channel =
898 (struct ltt_channel_struct *)buf->chan->private_data;
899 struct ltt_channel_buf_struct *ltt_buf = channel->buf;
900 int result;
901
902 //ust// #ifdef CONFIG_LTT_VMCORE
903 local_set(&ltt_buf->commit_seq[subbuf_idx], commit_count);
904 //ust// #endif
905
906 /* wakeup consumer */
907 result = write(ltt_buf->data_ready_fd_write, "1", 1);
908 if(result == -1) {
909 PERROR("write (in ltt_relay_buffer_flush)");
910 ERR("this should never happen!");
911 }
912 //ust// atomic_set(&ltt_buf->wakeup_readers, 1);
913 }
914
915 static struct dentry *ltt_create_buf_file_callback(struct rchan_buf *buf)
916 {
917 struct ltt_channel_struct *ltt_chan;
918 int err;
919 //ust// struct dentry *dentry;
920
921 ltt_chan = buf->chan->private_data;
922 err = ltt_relay_create_buffer(ltt_chan->trace, ltt_chan, buf, buf->chan->n_subbufs);
923 if (err)
924 return ERR_PTR(err);
925
926 //ust// dentry = debugfs_create_file(filename, mode, parent, buf,
927 //ust// &ltt_file_operations);
928 //ust// if (!dentry)
929 //ust// goto error;
930 //ust// return dentry;
931 return NULL; //ust//
932 //ust//error:
933 ltt_relay_destroy_buffer(ltt_chan);
934 return NULL;
935 }
936
937 //ust// static int ltt_remove_buf_file_callback(struct rchan_buf *buf)
938 //ust// {
939 //ust// //ust// struct rchan_buf *buf = dentry->d_inode->i_private;
940 //ust// struct ltt_channel_struct *ltt_chan = buf->chan->private_data;
941 //ust//
942 //ust// //ust// debugfs_remove(dentry);
943 //ust// ltt_relay_destroy_buffer(ltt_chan);
944 //ust//
945 //ust// return 0;
946 //ust// }
947
948 /*
949 * Wake writers :
950 *
951 * This must be done after the trace is removed from the RCU list so that there
952 * are no stalled writers.
953 */
954 //ust// static void ltt_relay_wake_writers(struct ltt_channel_buf_struct *ltt_buf)
955 //ust// {
956 //ust//
957 //ust// if (waitqueue_active(&ltt_buf->write_wait))
958 //ust// wake_up_interruptible(&ltt_buf->write_wait);
959 //ust// }
960
961 /*
962 * This function should not be called from NMI interrupt context
963 */
964 static notrace void ltt_buf_unfull(struct rchan_buf *buf,
965 unsigned int subbuf_idx,
966 long offset)
967 {
968 //ust// struct ltt_channel_struct *ltt_channel =
969 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
970 //ust// struct ltt_channel_buf_struct *ltt_buf = ltt_channel->buf;
971 //ust//
972 //ust// ltt_relay_wake_writers(ltt_buf);
973 }
974
975 /**
976 * ltt_open - open file op for ltt files
977 * @inode: opened inode
978 * @file: opened file
979 *
980 * Open implementation. Makes sure only one open instance of a buffer is
981 * done at a given moment.
982 */
983 //ust// static int ltt_open(struct inode *inode, struct file *file)
984 //ust// {
985 //ust// struct rchan_buf *buf = inode->i_private;
986 //ust// struct ltt_channel_struct *ltt_channel =
987 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
988 //ust// struct ltt_channel_buf_struct *ltt_buf =
989 //ust// percpu_ptr(ltt_channel->buf, buf->cpu);
990 //ust//
991 //ust// if (!atomic_long_add_unless(&ltt_buf->active_readers, 1, 1))
992 //ust// return -EBUSY;
993 //ust// return ltt_relay_file_operations.open(inode, file);
994 //ust// }
995
996 /**
997 * ltt_release - release file op for ltt files
998 * @inode: opened inode
999 * @file: opened file
1000 *
1001 * Release implementation.
1002 */
1003 //ust// static int ltt_release(struct inode *inode, struct file *file)
1004 //ust// {
1005 //ust// struct rchan_buf *buf = inode->i_private;
1006 //ust// struct ltt_channel_struct *ltt_channel =
1007 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
1008 //ust// struct ltt_channel_buf_struct *ltt_buf =
1009 //ust// percpu_ptr(ltt_channel->buf, buf->cpu);
1010 //ust// int ret;
1011 //ust//
1012 //ust// WARN_ON(atomic_long_read(&ltt_buf->active_readers) != 1);
1013 //ust// atomic_long_dec(&ltt_buf->active_readers);
1014 //ust// ret = ltt_relay_file_operations.release(inode, file);
1015 //ust// WARN_ON(ret);
1016 //ust// return ret;
1017 //ust// }
1018
1019 /**
1020 * ltt_poll - file op for ltt files
1021 * @filp: the file
1022 * @wait: poll table
1023 *
1024 * Poll implementation.
1025 */
1026 //ust// static unsigned int ltt_poll(struct file *filp, poll_table *wait)
1027 //ust// {
1028 //ust// unsigned int mask = 0;
1029 //ust// struct inode *inode = filp->f_dentry->d_inode;
1030 //ust// struct rchan_buf *buf = inode->i_private;
1031 //ust// struct ltt_channel_struct *ltt_channel =
1032 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
1033 //ust// struct ltt_channel_buf_struct *ltt_buf =
1034 //ust// percpu_ptr(ltt_channel->buf, buf->cpu);
1035 //ust//
1036 //ust// if (filp->f_mode & FMODE_READ) {
1037 //ust// poll_wait_set_exclusive(wait);
1038 //ust// poll_wait(filp, &buf->read_wait, wait);
1039 //ust//
1040 //ust// WARN_ON(atomic_long_read(&ltt_buf->active_readers) != 1);
1041 //ust// if (SUBBUF_TRUNC(local_read(&ltt_buf->offset),
1042 //ust// buf->chan)
1043 //ust// - SUBBUF_TRUNC(atomic_long_read(&ltt_buf->consumed),
1044 //ust// buf->chan)
1045 //ust// == 0) {
1046 //ust// if (buf->finalized)
1047 //ust// return POLLHUP;
1048 //ust// else
1049 //ust// return 0;
1050 //ust// } else {
1051 //ust// struct rchan *rchan =
1052 //ust// ltt_channel->trans_channel_data;
1053 //ust// if (SUBBUF_TRUNC(local_read(&ltt_buf->offset),
1054 //ust// buf->chan)
1055 //ust// - SUBBUF_TRUNC(atomic_long_read(
1056 //ust// &ltt_buf->consumed),
1057 //ust// buf->chan)
1058 //ust// >= rchan->alloc_size)
1059 //ust// return POLLPRI | POLLRDBAND;
1060 //ust// else
1061 //ust// return POLLIN | POLLRDNORM;
1062 //ust// }
1063 //ust// }
1064 //ust// return mask;
1065 //ust// }
1066
1067 int ltt_do_get_subbuf(struct rchan_buf *buf, struct ltt_channel_buf_struct *ltt_buf, long *pconsumed_old)
1068 {
1069 struct ltt_channel_struct *ltt_channel = (struct ltt_channel_struct *)buf->chan->private_data;
1070 long consumed_old, consumed_idx, commit_count, write_offset;
1071 consumed_old = atomic_long_read(&ltt_buf->consumed);
1072 consumed_idx = SUBBUF_INDEX(consumed_old, buf->chan);
1073 commit_count = local_read(&ltt_buf->commit_count[consumed_idx]);
1074 /*
1075 * Make sure we read the commit count before reading the buffer
1076 * data and the write offset. Correct consumed offset ordering
1077 * wrt commit count is insured by the use of cmpxchg to update
1078 * the consumed offset.
1079 */
1080 smp_rmb();
1081 write_offset = local_read(&ltt_buf->offset);
1082 /*
1083 * Check that the subbuffer we are trying to consume has been
1084 * already fully committed.
1085 */
1086 if (((commit_count - buf->chan->subbuf_size)
1087 & ltt_channel->commit_count_mask)
1088 - (BUFFER_TRUNC(consumed_old, buf->chan)
1089 >> ltt_channel->n_subbufs_order)
1090 != 0) {
1091 return -EAGAIN;
1092 }
1093 /*
1094 * Check that we are not about to read the same subbuffer in
1095 * which the writer head is.
1096 */
1097 if ((SUBBUF_TRUNC(write_offset, buf->chan)
1098 - SUBBUF_TRUNC(consumed_old, buf->chan))
1099 == 0) {
1100 return -EAGAIN;
1101 }
1102
1103 *pconsumed_old = consumed_old;
1104 return 0;
1105 }
1106
1107 int ltt_do_put_subbuf(struct rchan_buf *buf, struct ltt_channel_buf_struct *ltt_buf, u32 uconsumed_old)
1108 {
1109 long consumed_new, consumed_old;
1110
1111 consumed_old = atomic_long_read(&ltt_buf->consumed);
1112 consumed_old = consumed_old & (~0xFFFFFFFFL);
1113 consumed_old = consumed_old | uconsumed_old;
1114 consumed_new = SUBBUF_ALIGN(consumed_old, buf->chan);
1115
1116 //ust// spin_lock(&ltt_buf->full_lock);
1117 if (atomic_long_cmpxchg(&ltt_buf->consumed, consumed_old,
1118 consumed_new)
1119 != consumed_old) {
1120 /* We have been pushed by the writer : the last
1121 * buffer read _is_ corrupted! It can also
1122 * happen if this is a buffer we never got. */
1123 //ust// spin_unlock(&ltt_buf->full_lock);
1124 return -EIO;
1125 } else {
1126 /* tell the client that buffer is now unfull */
1127 int index;
1128 long data;
1129 index = SUBBUF_INDEX(consumed_old, buf->chan);
1130 data = BUFFER_OFFSET(consumed_old, buf->chan);
1131 ltt_buf_unfull(buf, index, data);
1132 //ust// spin_unlock(&ltt_buf->full_lock);
1133 }
1134 return 0;
1135 }
1136
1137 /**
1138 * ltt_ioctl - control on the debugfs file
1139 *
1140 * @inode: the inode
1141 * @filp: the file
1142 * @cmd: the command
1143 * @arg: command arg
1144 *
1145 * This ioctl implements three commands necessary for a minimal
1146 * producer/consumer implementation :
1147 * RELAY_GET_SUBBUF
1148 * Get the next sub buffer that can be read. It never blocks.
1149 * RELAY_PUT_SUBBUF
1150 * Release the currently read sub-buffer. Parameter is the last
1151 * put subbuffer (returned by GET_SUBBUF).
1152 * RELAY_GET_N_BUBBUFS
1153 * returns the number of sub buffers in the per cpu channel.
1154 * RELAY_GET_SUBBUF_SIZE
1155 * returns the size of the sub buffers.
1156 */
1157 //ust// static int ltt_ioctl(struct inode *inode, struct file *filp,
1158 //ust// unsigned int cmd, unsigned long arg)
1159 //ust// {
1160 //ust// struct rchan_buf *buf = inode->i_private;
1161 //ust// struct ltt_channel_struct *ltt_channel =
1162 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
1163 //ust// struct ltt_channel_buf_struct *ltt_buf =
1164 //ust// percpu_ptr(ltt_channel->buf, buf->cpu);
1165 //ust// u32 __user *argp = (u32 __user *)arg;
1166 //ust//
1167 //ust// WARN_ON(atomic_long_read(&ltt_buf->active_readers) != 1);
1168 //ust// switch (cmd) {
1169 //ust// case RELAY_GET_SUBBUF:
1170 //ust// {
1171 //ust// int ret;
1172 //ust// ret = ltt_do_get_subbuf(buf, ltt_buf, &consumed_old);
1173 //ust// if(ret < 0)
1174 //ust// return ret;
1175 //ust// return put_user((u32)consumed_old, argp);
1176 //ust// }
1177 //ust// case RELAY_PUT_SUBBUF:
1178 //ust// {
1179 //ust// int ret;
1180 //ust// u32 uconsumed_old;
1181 //ust// ret = get_user(uconsumed_old, argp);
1182 //ust// if (ret)
1183 //ust// return ret; /* will return -EFAULT */
1184 //ust// return ltt_do_put_subbuf(buf, ltt_buf, uconsumed_old);
1185 //ust// }
1186 //ust// case RELAY_GET_N_SUBBUFS:
1187 //ust// return put_user((u32)buf->chan->n_subbufs, argp);
1188 //ust// break;
1189 //ust// case RELAY_GET_SUBBUF_SIZE:
1190 //ust// return put_user((u32)buf->chan->subbuf_size, argp);
1191 //ust// break;
1192 //ust// default:
1193 //ust// return -ENOIOCTLCMD;
1194 //ust// }
1195 //ust// return 0;
1196 //ust// }
1197
1198 //ust// #ifdef CONFIG_COMPAT
1199 //ust// static long ltt_compat_ioctl(struct file *file, unsigned int cmd,
1200 //ust// unsigned long arg)
1201 //ust// {
1202 //ust// long ret = -ENOIOCTLCMD;
1203 //ust//
1204 //ust// lock_kernel();
1205 //ust// ret = ltt_ioctl(file->f_dentry->d_inode, file, cmd, arg);
1206 //ust// unlock_kernel();
1207 //ust//
1208 //ust// return ret;
1209 //ust// }
1210 //ust// #endif
1211
1212 //ust// static void ltt_relay_pipe_buf_release(struct pipe_inode_info *pipe,
1213 //ust// struct pipe_buffer *pbuf)
1214 //ust// {
1215 //ust// }
1216 //ust//
1217 //ust// static struct pipe_buf_operations ltt_relay_pipe_buf_ops = {
1218 //ust// .can_merge = 0,
1219 //ust// .map = generic_pipe_buf_map,
1220 //ust// .unmap = generic_pipe_buf_unmap,
1221 //ust// .confirm = generic_pipe_buf_confirm,
1222 //ust// .release = ltt_relay_pipe_buf_release,
1223 //ust// .steal = generic_pipe_buf_steal,
1224 //ust// .get = generic_pipe_buf_get,
1225 //ust// };
1226
1227 //ust// static void ltt_relay_page_release(struct splice_pipe_desc *spd, unsigned int i)
1228 //ust// {
1229 //ust// }
1230
1231 /*
1232 * subbuf_splice_actor - splice up to one subbuf's worth of data
1233 */
1234 //ust// static int subbuf_splice_actor(struct file *in,
1235 //ust// loff_t *ppos,
1236 //ust// struct pipe_inode_info *pipe,
1237 //ust// size_t len,
1238 //ust// unsigned int flags)
1239 //ust// {
1240 //ust// struct rchan_buf *buf = in->private_data;
1241 //ust// struct ltt_channel_struct *ltt_channel =
1242 //ust// (struct ltt_channel_struct *)buf->chan->private_data;
1243 //ust// struct ltt_channel_buf_struct *ltt_buf =
1244 //ust// percpu_ptr(ltt_channel->buf, buf->cpu);
1245 //ust// unsigned int poff, subbuf_pages, nr_pages;
1246 //ust// struct page *pages[PIPE_BUFFERS];
1247 //ust// struct partial_page partial[PIPE_BUFFERS];
1248 //ust// struct splice_pipe_desc spd = {
1249 //ust// .pages = pages,
1250 //ust// .nr_pages = 0,
1251 //ust// .partial = partial,
1252 //ust// .flags = flags,
1253 //ust// .ops = &ltt_relay_pipe_buf_ops,
1254 //ust// .spd_release = ltt_relay_page_release,
1255 //ust// };
1256 //ust// long consumed_old, consumed_idx, roffset;
1257 //ust// unsigned long bytes_avail;
1258 //ust//
1259 //ust// /*
1260 //ust// * Check that a GET_SUBBUF ioctl has been done before.
1261 //ust// */
1262 //ust// WARN_ON(atomic_long_read(&ltt_buf->active_readers) != 1);
1263 //ust// consumed_old = atomic_long_read(&ltt_buf->consumed);
1264 //ust// consumed_old += *ppos;
1265 //ust// consumed_idx = SUBBUF_INDEX(consumed_old, buf->chan);
1266 //ust//
1267 //ust// /*
1268 //ust// * Adjust read len, if longer than what is available
1269 //ust// */
1270 //ust// bytes_avail = SUBBUF_TRUNC(local_read(&ltt_buf->offset), buf->chan)
1271 //ust// - consumed_old;
1272 //ust// WARN_ON(bytes_avail > buf->chan->alloc_size);
1273 //ust// len = min_t(size_t, len, bytes_avail);
1274 //ust// subbuf_pages = bytes_avail >> PAGE_SHIFT;
1275 //ust// nr_pages = min_t(unsigned int, subbuf_pages, PIPE_BUFFERS);
1276 //ust// roffset = consumed_old & PAGE_MASK;
1277 //ust// poff = consumed_old & ~PAGE_MASK;
1278 //ust// printk_dbg(KERN_DEBUG "SPLICE actor len %zu pos %zd write_pos %ld\n",
1279 //ust// len, (ssize_t)*ppos, local_read(&ltt_buf->offset));
1280 //ust//
1281 //ust// for (; spd.nr_pages < nr_pages; spd.nr_pages++) {
1282 //ust// unsigned int this_len;
1283 //ust// struct buf_page *page;
1284 //ust//
1285 //ust// if (!len)
1286 //ust// break;
1287 //ust// printk_dbg(KERN_DEBUG "SPLICE actor loop len %zu roffset %ld\n",
1288 //ust// len, roffset);
1289 //ust//
1290 //ust// this_len = PAGE_SIZE - poff;
1291 //ust// page = ltt_relay_read_get_page(buf, roffset);
1292 //ust// spd.pages[spd.nr_pages] = page->page;
1293 //ust// spd.partial[spd.nr_pages].offset = poff;
1294 //ust// spd.partial[spd.nr_pages].len = this_len;
1295 //ust//
1296 //ust// poff = 0;
1297 //ust// roffset += PAGE_SIZE;
1298 //ust// len -= this_len;
1299 //ust// }
1300 //ust//
1301 //ust// if (!spd.nr_pages)
1302 //ust// return 0;
1303 //ust//
1304 //ust// return splice_to_pipe(pipe, &spd);
1305 //ust// }
1306
1307 //ust// static ssize_t ltt_relay_file_splice_read(struct file *in,
1308 //ust// loff_t *ppos,
1309 //ust// struct pipe_inode_info *pipe,
1310 //ust// size_t len,
1311 //ust// unsigned int flags)
1312 //ust// {
1313 //ust// ssize_t spliced;
1314 //ust// int ret;
1315 //ust//
1316 //ust// ret = 0;
1317 //ust// spliced = 0;
1318 //ust//
1319 //ust// printk_dbg(KERN_DEBUG "SPLICE read len %zu pos %zd\n",
1320 //ust// len, (ssize_t)*ppos);
1321 //ust// while (len && !spliced) {
1322 //ust// ret = subbuf_splice_actor(in, ppos, pipe, len, flags);
1323 //ust// printk_dbg(KERN_DEBUG "SPLICE read loop ret %d\n", ret);
1324 //ust// if (ret < 0)
1325 //ust// break;
1326 //ust// else if (!ret) {
1327 //ust// if (flags & SPLICE_F_NONBLOCK)
1328 //ust// ret = -EAGAIN;
1329 //ust// break;
1330 //ust// }
1331 //ust//
1332 //ust// *ppos += ret;
1333 //ust// if (ret > len)
1334 //ust// len = 0;
1335 //ust// else
1336 //ust// len -= ret;
1337 //ust// spliced += ret;
1338 //ust// }
1339 //ust//
1340 //ust// if (spliced)
1341 //ust// return spliced;
1342 //ust//
1343 //ust// return ret;
1344 //ust// }
1345
1346 static void ltt_relay_print_subbuffer_errors(
1347 struct ltt_channel_struct *ltt_chan,
1348 long cons_off)
1349 {
1350 struct rchan *rchan = ltt_chan->trans_channel_data;
1351 struct ltt_channel_buf_struct *ltt_buf = ltt_chan->buf;
1352 long cons_idx, commit_count, write_offset;
1353
1354 cons_idx = SUBBUF_INDEX(cons_off, rchan);
1355 commit_count = local_read(&ltt_buf->commit_count[cons_idx]);
1356 /*
1357 * No need to order commit_count and write_offset reads because we
1358 * execute after trace is stopped when there are no readers left.
1359 */
1360 write_offset = local_read(&ltt_buf->offset);
1361 printk(KERN_WARNING
1362 "LTT : unread channel %s offset is %ld "
1363 "and cons_off : %ld\n",
1364 ltt_chan->channel_name, write_offset, cons_off);
1365 /* Check each sub-buffer for non filled commit count */
1366 if (((commit_count - rchan->subbuf_size) & ltt_chan->commit_count_mask)
1367 - (BUFFER_TRUNC(cons_off, rchan) >> ltt_chan->n_subbufs_order)
1368 != 0)
1369 printk(KERN_ALERT
1370 "LTT : %s : subbuffer %lu has non filled "
1371 "commit count %lu.\n",
1372 ltt_chan->channel_name, cons_idx, commit_count);
1373 printk(KERN_ALERT "LTT : %s : commit count : %lu, subbuf size %zd\n",
1374 ltt_chan->channel_name, commit_count,
1375 rchan->subbuf_size);
1376 }
1377
1378 static void ltt_relay_print_errors(struct ltt_trace_struct *trace,
1379 struct ltt_channel_struct *ltt_chan)
1380 {
1381 struct rchan *rchan = ltt_chan->trans_channel_data;
1382 struct ltt_channel_buf_struct *ltt_buf = ltt_chan->buf;
1383 long cons_off;
1384
1385 for (cons_off = atomic_long_read(&ltt_buf->consumed);
1386 (SUBBUF_TRUNC(local_read(&ltt_buf->offset),
1387 rchan)
1388 - cons_off) > 0;
1389 cons_off = SUBBUF_ALIGN(cons_off, rchan))
1390 ltt_relay_print_subbuffer_errors(ltt_chan, cons_off);
1391 }
1392
1393 static void ltt_relay_print_buffer_errors(struct ltt_channel_struct *ltt_chan)
1394 {
1395 struct ltt_trace_struct *trace = ltt_chan->trace;
1396 struct ltt_channel_buf_struct *ltt_buf = ltt_chan->buf;
1397
1398 if (local_read(&ltt_buf->events_lost))
1399 printk(KERN_ALERT
1400 "LTT : %s : %ld events lost "
1401 "in %s channel.\n",
1402 ltt_chan->channel_name,
1403 local_read(&ltt_buf->events_lost),
1404 ltt_chan->channel_name);
1405 if (local_read(&ltt_buf->corrupted_subbuffers))
1406 printk(KERN_ALERT
1407 "LTT : %s : %ld corrupted subbuffers "
1408 "in %s channel.\n",
1409 ltt_chan->channel_name,
1410 local_read(&ltt_buf->corrupted_subbuffers),
1411 ltt_chan->channel_name);
1412
1413 ltt_relay_print_errors(trace, ltt_chan);
1414 }
1415
1416 static void ltt_relay_remove_dirs(struct ltt_trace_struct *trace)
1417 {
1418 //ust// debugfs_remove(trace->dentry.trace_root);
1419 }
1420
1421 static void ltt_relay_release_channel(struct kref *kref)
1422 {
1423 struct ltt_channel_struct *ltt_chan = container_of(kref,
1424 struct ltt_channel_struct, kref);
1425 free(ltt_chan->buf);
1426 }
1427
1428 /*
1429 * Create ltt buffer.
1430 */
1431 //ust// static int ltt_relay_create_buffer(struct ltt_trace_struct *trace,
1432 //ust// struct ltt_channel_struct *ltt_chan, struct rchan_buf *buf,
1433 //ust// unsigned int cpu, unsigned int n_subbufs)
1434 //ust// {
1435 //ust// struct ltt_channel_buf_struct *ltt_buf =
1436 //ust// percpu_ptr(ltt_chan->buf, cpu);
1437 //ust// unsigned int j;
1438 //ust//
1439 //ust// ltt_buf->commit_count =
1440 //ust// kzalloc_node(sizeof(ltt_buf->commit_count) * n_subbufs,
1441 //ust// GFP_KERNEL, cpu_to_node(cpu));
1442 //ust// if (!ltt_buf->commit_count)
1443 //ust// return -ENOMEM;
1444 //ust// kref_get(&trace->kref);
1445 //ust// kref_get(&trace->ltt_transport_kref);
1446 //ust// kref_get(&ltt_chan->kref);
1447 //ust// local_set(&ltt_buf->offset, ltt_subbuffer_header_size());
1448 //ust// atomic_long_set(&ltt_buf->consumed, 0);
1449 //ust// atomic_long_set(&ltt_buf->active_readers, 0);
1450 //ust// for (j = 0; j < n_subbufs; j++)
1451 //ust// local_set(&ltt_buf->commit_count[j], 0);
1452 //ust// init_waitqueue_head(&ltt_buf->write_wait);
1453 //ust// atomic_set(&ltt_buf->wakeup_readers, 0);
1454 //ust// spin_lock_init(&ltt_buf->full_lock);
1455 //ust//
1456 //ust// ltt_buffer_begin_callback(buf, trace->start_tsc, 0);
1457 //ust// /* atomic_add made on local variable on data that belongs to
1458 //ust// * various CPUs : ok because tracing not started (for this cpu). */
1459 //ust// local_add(ltt_subbuffer_header_size(), &ltt_buf->commit_count[0]);
1460 //ust//
1461 //ust// local_set(&ltt_buf->events_lost, 0);
1462 //ust// local_set(&ltt_buf->corrupted_subbuffers, 0);
1463 //ust//
1464 //ust// return 0;
1465 //ust// }
1466
1467 static int ltt_relay_create_buffer(struct ltt_trace_struct *trace,
1468 struct ltt_channel_struct *ltt_chan, struct rchan_buf *buf,
1469 unsigned int n_subbufs)
1470 {
1471 struct ltt_channel_buf_struct *ltt_buf = ltt_chan->buf;
1472 unsigned int j;
1473 int fds[2];
1474 int result;
1475
1476 ltt_buf->commit_count =
1477 zmalloc(sizeof(ltt_buf->commit_count) * n_subbufs);
1478 if (!ltt_buf->commit_count)
1479 return -ENOMEM;
1480 kref_get(&trace->kref);
1481 kref_get(&trace->ltt_transport_kref);
1482 kref_get(&ltt_chan->kref);
1483 local_set(&ltt_buf->offset, ltt_subbuffer_header_size());
1484 atomic_long_set(&ltt_buf->consumed, 0);
1485 atomic_long_set(&ltt_buf->active_readers, 0);
1486 for (j = 0; j < n_subbufs; j++)
1487 local_set(&ltt_buf->commit_count[j], 0);
1488 //ust// init_waitqueue_head(&ltt_buf->write_wait);
1489 //ust// atomic_set(&ltt_buf->wakeup_readers, 0);
1490 //ust// spin_lock_init(&ltt_buf->full_lock);
1491
1492 ltt_buffer_begin_callback(buf, trace->start_tsc, 0);
1493
1494 local_add(ltt_subbuffer_header_size(), &ltt_buf->commit_count[0]);
1495
1496 local_set(&ltt_buf->events_lost, 0);
1497 local_set(&ltt_buf->corrupted_subbuffers, 0);
1498
1499 result = pipe(fds);
1500 if(result == -1) {
1501 PERROR("pipe");
1502 return -1;
1503 }
1504 ltt_buf->data_ready_fd_read = fds[0];
1505 ltt_buf->data_ready_fd_write = fds[1];
1506
1507 /* FIXME: do we actually need this? */
1508 result = fcntl(fds[0], F_SETFL, O_NONBLOCK);
1509 if(result == -1) {
1510 PERROR("fcntl");
1511 }
1512
1513 //ust// ltt_buf->commit_seq = malloc(sizeof(ltt_buf->commit_seq) * n_subbufs);
1514 //ust// if(!ltt_buf->commit_seq) {
1515 //ust// return -1;
1516 //ust// }
1517
1518 /* FIXME: decrementally destroy on error */
1519
1520 return 0;
1521 }
1522
1523 static void ltt_relay_destroy_buffer(struct ltt_channel_struct *ltt_chan)
1524 {
1525 struct ltt_trace_struct *trace = ltt_chan->trace;
1526 struct ltt_channel_buf_struct *ltt_buf = ltt_chan->buf;
1527
1528 kref_put(&ltt_chan->trace->ltt_transport_kref,
1529 ltt_release_transport);
1530 ltt_relay_print_buffer_errors(ltt_chan);
1531 //ust// free(ltt_buf->commit_seq);
1532 kfree(ltt_buf->commit_count);
1533 ltt_buf->commit_count = NULL;
1534 kref_put(&ltt_chan->kref, ltt_relay_release_channel);
1535 kref_put(&trace->kref, ltt_release_trace);
1536 //ust// wake_up_interruptible(&trace->kref_wq);
1537 }
1538
1539 static void ltt_chan_alloc_ltt_buf(struct ltt_channel_struct *ltt_chan)
1540 {
1541 void *ptr;
1542 int result;
1543
1544 /* Get one page */
1545 /* FIXME: increase size if we have a seq_commit array that overflows the page */
1546 size_t size = PAGE_ALIGN(1);
1547
1548 result = ltt_chan->buf_shmid = shmget(getpid(), size, IPC_CREAT | IPC_EXCL | 0700);
1549 if(ltt_chan->buf_shmid == -1) {
1550 PERROR("shmget");
1551 return;
1552 }
1553
1554 ptr = shmat(ltt_chan->buf_shmid, NULL, 0);
1555 if(ptr == (void *) -1) {
1556 perror("shmat");
1557 goto destroy_shmem;
1558 }
1559
1560 /* Already mark the shared memory for destruction. This will occur only
1561 * when all users have detached.
1562 */
1563 result = shmctl(ltt_chan->buf_shmid, IPC_RMID, NULL);
1564 if(result == -1) {
1565 perror("shmctl");
1566 return;
1567 }
1568
1569 ltt_chan->buf = ptr;
1570
1571 return;
1572
1573 destroy_shmem:
1574 result = shmctl(ltt_chan->buf_shmid, IPC_RMID, NULL);
1575 if(result == -1) {
1576 perror("shmctl");
1577 }
1578
1579 return;
1580 }
1581
1582 /*
1583 * Create channel.
1584 */
1585 static int ltt_relay_create_channel(const char *trace_name,
1586 struct ltt_trace_struct *trace, struct dentry *dir,
1587 const char *channel_name, struct ltt_channel_struct *ltt_chan,
1588 unsigned int subbuf_size, unsigned int n_subbufs,
1589 int overwrite)
1590 {
1591 char *tmpname;
1592 unsigned int tmpname_len;
1593 int err = 0;
1594
1595 tmpname = kmalloc(PATH_MAX, GFP_KERNEL);
1596 if (!tmpname)
1597 return EPERM;
1598 if (overwrite) {
1599 strncpy(tmpname, LTT_FLIGHT_PREFIX, PATH_MAX-1);
1600 strncat(tmpname, channel_name,
1601 PATH_MAX-1-sizeof(LTT_FLIGHT_PREFIX));
1602 } else {
1603 strncpy(tmpname, channel_name, PATH_MAX-1);
1604 }
1605 strncat(tmpname, "_", PATH_MAX-1-strlen(tmpname));
1606
1607 kref_init(&ltt_chan->kref);
1608
1609 ltt_chan->trace = trace;
1610 ltt_chan->buffer_begin = ltt_buffer_begin_callback;
1611 ltt_chan->buffer_end = ltt_buffer_end_callback;
1612 ltt_chan->overwrite = overwrite;
1613 ltt_chan->n_subbufs_order = get_count_order(n_subbufs);
1614 ltt_chan->commit_count_mask = (~0UL >> ltt_chan->n_subbufs_order);
1615 //ust// ltt_chan->buf = percpu_alloc_mask(sizeof(struct ltt_channel_buf_struct), GFP_KERNEL, cpu_possible_map);
1616
1617 ltt_chan_alloc_ltt_buf(ltt_chan);
1618
1619 //ust// ltt_chan->buf = malloc(sizeof(struct ltt_channel_buf_struct));
1620 if (!ltt_chan->buf)
1621 goto alloc_error;
1622 ltt_chan->trans_channel_data = ltt_relay_open(tmpname,
1623 dir,
1624 subbuf_size,
1625 n_subbufs,
1626 ltt_chan);
1627 tmpname_len = strlen(tmpname);
1628 if (tmpname_len > 0) {
1629 /* Remove final _ for pretty printing */
1630 tmpname[tmpname_len-1] = '\0';
1631 }
1632 if (ltt_chan->trans_channel_data == NULL) {
1633 printk(KERN_ERR "LTT : Can't open %s channel for trace %s\n",
1634 tmpname, trace_name);
1635 goto relay_open_error;
1636 }
1637
1638 err = 0;
1639 goto end;
1640
1641 relay_open_error:
1642 //ust// percpu_free(ltt_chan->buf);
1643 alloc_error:
1644 err = EPERM;
1645 end:
1646 kfree(tmpname);
1647 return err;
1648 }
1649
1650 static int ltt_relay_create_dirs(struct ltt_trace_struct *new_trace)
1651 {
1652 //ust// new_trace->dentry.trace_root = debugfs_create_dir(new_trace->trace_name,
1653 //ust// get_ltt_root());
1654 //ust// if (new_trace->dentry.trace_root == NULL) {
1655 //ust// printk(KERN_ERR "LTT : Trace directory name %s already taken\n",
1656 //ust// new_trace->trace_name);
1657 //ust// return EEXIST;
1658 //ust// }
1659
1660 //ust// new_trace->callbacks.create_buf_file = ltt_create_buf_file_callback;
1661 //ust// new_trace->callbacks.remove_buf_file = ltt_remove_buf_file_callback;
1662
1663 return 0;
1664 }
1665
1666 /*
1667 * LTTng channel flush function.
1668 *
1669 * Must be called when no tracing is active in the channel, because of
1670 * accesses across CPUs.
1671 */
1672 static notrace void ltt_relay_buffer_flush(struct rchan_buf *buf)
1673 {
1674 struct ltt_channel_struct *channel =
1675 (struct ltt_channel_struct *)buf->chan->private_data;
1676 struct ltt_channel_buf_struct *ltt_buf = channel->buf;
1677 int result;
1678
1679 buf->finalized = 1;
1680 ltt_force_switch(buf, FORCE_FLUSH);
1681
1682 result = write(ltt_buf->data_ready_fd_write, "1", 1);
1683 if(result == -1) {
1684 PERROR("write (in ltt_relay_buffer_flush)");
1685 ERR("this should never happen!");
1686 }
1687 }
1688
1689 static void ltt_relay_async_wakeup_chan(struct ltt_channel_struct *ltt_channel)
1690 {
1691 //ust// unsigned int i;
1692 //ust// struct rchan *rchan = ltt_channel->trans_channel_data;
1693 //ust//
1694 //ust// for_each_possible_cpu(i) {
1695 //ust// struct ltt_channel_buf_struct *ltt_buf =
1696 //ust// percpu_ptr(ltt_channel->buf, i);
1697 //ust//
1698 //ust// if (atomic_read(&ltt_buf->wakeup_readers) == 1) {
1699 //ust// atomic_set(&ltt_buf->wakeup_readers, 0);
1700 //ust// wake_up_interruptible(&rchan->buf[i]->read_wait);
1701 //ust// }
1702 //ust// }
1703 }
1704
1705 static void ltt_relay_finish_buffer(struct ltt_channel_struct *ltt_channel)
1706 {
1707 struct rchan *rchan = ltt_channel->trans_channel_data;
1708 // int result;
1709
1710 if (rchan->buf) {
1711 struct ltt_channel_buf_struct *ltt_buf = ltt_channel->buf;
1712 ltt_relay_buffer_flush(rchan->buf);
1713 //ust// ltt_relay_wake_writers(ltt_buf);
1714 /* closing the pipe tells the consumer the buffer is finished */
1715
1716 //result = write(ltt_buf->data_ready_fd_write, "D", 1);
1717 //if(result == -1) {
1718 // PERROR("write (in ltt_relay_finish_buffer)");
1719 // ERR("this should never happen!");
1720 //}
1721 close(ltt_buf->data_ready_fd_write);
1722 }
1723 }
1724
1725
1726 static void ltt_relay_finish_channel(struct ltt_channel_struct *ltt_channel)
1727 {
1728 //ust// unsigned int i;
1729
1730 //ust// for_each_possible_cpu(i)
1731 ltt_relay_finish_buffer(ltt_channel);
1732 }
1733
1734 static void ltt_relay_remove_channel(struct ltt_channel_struct *channel)
1735 {
1736 struct rchan *rchan = channel->trans_channel_data;
1737
1738 ltt_relay_close(rchan);
1739 kref_put(&channel->kref, ltt_relay_release_channel);
1740 }
1741
1742 struct ltt_reserve_switch_offsets {
1743 long begin, end, old;
1744 long begin_switch, end_switch_current, end_switch_old;
1745 long commit_count, reserve_commit_diff;
1746 size_t before_hdr_pad, size;
1747 };
1748
1749 /*
1750 * Returns :
1751 * 0 if ok
1752 * !0 if execution must be aborted.
1753 */
1754 static inline int ltt_relay_try_reserve(
1755 struct ltt_channel_struct *ltt_channel,
1756 struct ltt_channel_buf_struct *ltt_buf, struct rchan *rchan,
1757 struct rchan_buf *buf,
1758 struct ltt_reserve_switch_offsets *offsets, size_t data_size,
1759 u64 *tsc, unsigned int *rflags, int largest_align)
1760 {
1761 offsets->begin = local_read(&ltt_buf->offset);
1762 offsets->old = offsets->begin;
1763 offsets->begin_switch = 0;
1764 offsets->end_switch_current = 0;
1765 offsets->end_switch_old = 0;
1766
1767 *tsc = trace_clock_read64();
1768 if (last_tsc_overflow(ltt_buf, *tsc))
1769 *rflags = LTT_RFLAG_ID_SIZE_TSC;
1770
1771 if (SUBBUF_OFFSET(offsets->begin, buf->chan) == 0) {
1772 offsets->begin_switch = 1; /* For offsets->begin */
1773 } else {
1774 offsets->size = ltt_get_header_size(ltt_channel,
1775 offsets->begin, data_size,
1776 &offsets->before_hdr_pad, *rflags);
1777 offsets->size += ltt_align(offsets->begin + offsets->size,
1778 largest_align)
1779 + data_size;
1780 if ((SUBBUF_OFFSET(offsets->begin, buf->chan) + offsets->size)
1781 > buf->chan->subbuf_size) {
1782 offsets->end_switch_old = 1; /* For offsets->old */
1783 offsets->begin_switch = 1; /* For offsets->begin */
1784 }
1785 }
1786 if (offsets->begin_switch) {
1787 long subbuf_index;
1788
1789 if (offsets->end_switch_old)
1790 offsets->begin = SUBBUF_ALIGN(offsets->begin,
1791 buf->chan);
1792 offsets->begin = offsets->begin + ltt_subbuffer_header_size();
1793 /* Test new buffer integrity */
1794 subbuf_index = SUBBUF_INDEX(offsets->begin, buf->chan);
1795 offsets->reserve_commit_diff =
1796 (BUFFER_TRUNC(offsets->begin, buf->chan)
1797 >> ltt_channel->n_subbufs_order)
1798 - (local_read(&ltt_buf->commit_count[subbuf_index])
1799 & ltt_channel->commit_count_mask);
1800 if (offsets->reserve_commit_diff == 0) {
1801 long consumed;
1802
1803 consumed = atomic_long_read(&ltt_buf->consumed);
1804
1805 /* Next buffer not corrupted. */
1806 if (!ltt_channel->overwrite &&
1807 (SUBBUF_TRUNC(offsets->begin, buf->chan)
1808 - SUBBUF_TRUNC(consumed, buf->chan))
1809 >= rchan->alloc_size) {
1810
1811 long consumed_idx = SUBBUF_INDEX(consumed, buf->chan);
1812 long commit_count = local_read(&ltt_buf->commit_count[consumed_idx]);
1813 if(((commit_count - buf->chan->subbuf_size) & ltt_channel->commit_count_mask) - (BUFFER_TRUNC(consumed, buf->chan) >> ltt_channel->n_subbufs_order) != 0) {
1814 WARN("Event dropped. Caused by non-committed event.");
1815 }
1816 else {
1817 WARN("Event dropped. Caused by non-consumed buffer.");
1818 }
1819 /*
1820 * We do not overwrite non consumed buffers
1821 * and we are full : event is lost.
1822 */
1823 local_inc(&ltt_buf->events_lost);
1824 return -1;
1825 } else {
1826 /*
1827 * next buffer not corrupted, we are either in
1828 * overwrite mode or the buffer is not full.
1829 * It's safe to write in this new subbuffer.
1830 */
1831 }
1832 } else {
1833 /*
1834 * Next subbuffer corrupted. Force pushing reader even
1835 * in normal mode. It's safe to write in this new
1836 * subbuffer.
1837 */
1838 }
1839 offsets->size = ltt_get_header_size(ltt_channel,
1840 offsets->begin, data_size,
1841 &offsets->before_hdr_pad, *rflags);
1842 offsets->size += ltt_align(offsets->begin + offsets->size,
1843 largest_align)
1844 + data_size;
1845 if ((SUBBUF_OFFSET(offsets->begin, buf->chan) + offsets->size)
1846 > buf->chan->subbuf_size) {
1847 /*
1848 * Event too big for subbuffers, report error, don't
1849 * complete the sub-buffer switch.
1850 */
1851 local_inc(&ltt_buf->events_lost);
1852 return -1;
1853 } else {
1854 /*
1855 * We just made a successful buffer switch and the event
1856 * fits in the new subbuffer. Let's write.
1857 */
1858 }
1859 } else {
1860 /*
1861 * Event fits in the current buffer and we are not on a switch
1862 * boundary. It's safe to write.
1863 */
1864 }
1865 offsets->end = offsets->begin + offsets->size;
1866
1867 if ((SUBBUF_OFFSET(offsets->end, buf->chan)) == 0) {
1868 /*
1869 * The offset_end will fall at the very beginning of the next
1870 * subbuffer.
1871 */
1872 offsets->end_switch_current = 1; /* For offsets->begin */
1873 }
1874 return 0;
1875 }
1876
1877 /*
1878 * Returns :
1879 * 0 if ok
1880 * !0 if execution must be aborted.
1881 */
1882 static inline int ltt_relay_try_switch(
1883 enum force_switch_mode mode,
1884 struct ltt_channel_struct *ltt_channel,
1885 struct ltt_channel_buf_struct *ltt_buf, struct rchan *rchan,
1886 struct rchan_buf *buf,
1887 struct ltt_reserve_switch_offsets *offsets,
1888 u64 *tsc)
1889 {
1890 long subbuf_index;
1891
1892 offsets->begin = local_read(&ltt_buf->offset);
1893 offsets->old = offsets->begin;
1894 offsets->begin_switch = 0;
1895 offsets->end_switch_old = 0;
1896
1897 *tsc = trace_clock_read64();
1898
1899 if (SUBBUF_OFFSET(offsets->begin, buf->chan) != 0) {
1900 offsets->begin = SUBBUF_ALIGN(offsets->begin, buf->chan);
1901 offsets->end_switch_old = 1;
1902 } else {
1903 /* we do not have to switch : buffer is empty */
1904 return -1;
1905 }
1906 if (mode == FORCE_ACTIVE)
1907 offsets->begin += ltt_subbuffer_header_size();
1908 /*
1909 * Always begin_switch in FORCE_ACTIVE mode.
1910 * Test new buffer integrity
1911 */
1912 subbuf_index = SUBBUF_INDEX(offsets->begin, buf->chan);
1913 offsets->reserve_commit_diff =
1914 (BUFFER_TRUNC(offsets->begin, buf->chan)
1915 >> ltt_channel->n_subbufs_order)
1916 - (local_read(&ltt_buf->commit_count[subbuf_index])
1917 & ltt_channel->commit_count_mask);
1918 if (offsets->reserve_commit_diff == 0) {
1919 /* Next buffer not corrupted. */
1920 if (mode == FORCE_ACTIVE
1921 && !ltt_channel->overwrite
1922 && offsets->begin - atomic_long_read(&ltt_buf->consumed)
1923 >= rchan->alloc_size) {
1924 /*
1925 * We do not overwrite non consumed buffers and we are
1926 * full : ignore switch while tracing is active.
1927 */
1928 return -1;
1929 }
1930 } else {
1931 /*
1932 * Next subbuffer corrupted. Force pushing reader even in normal
1933 * mode
1934 */
1935 }
1936 offsets->end = offsets->begin;
1937 return 0;
1938 }
1939
1940 static inline void ltt_reserve_push_reader(
1941 struct ltt_channel_struct *ltt_channel,
1942 struct ltt_channel_buf_struct *ltt_buf,
1943 struct rchan *rchan,
1944 struct rchan_buf *buf,
1945 struct ltt_reserve_switch_offsets *offsets)
1946 {
1947 long consumed_old, consumed_new;
1948
1949 do {
1950 consumed_old = atomic_long_read(&ltt_buf->consumed);
1951 /*
1952 * If buffer is in overwrite mode, push the reader consumed
1953 * count if the write position has reached it and we are not
1954 * at the first iteration (don't push the reader farther than
1955 * the writer). This operation can be done concurrently by many
1956 * writers in the same buffer, the writer being at the farthest
1957 * write position sub-buffer index in the buffer being the one
1958 * which will win this loop.
1959 * If the buffer is not in overwrite mode, pushing the reader
1960 * only happens if a sub-buffer is corrupted.
1961 */
1962 if ((SUBBUF_TRUNC(offsets->end-1, buf->chan)
1963 - SUBBUF_TRUNC(consumed_old, buf->chan))
1964 >= rchan->alloc_size)
1965 consumed_new = SUBBUF_ALIGN(consumed_old, buf->chan);
1966 else {
1967 consumed_new = consumed_old;
1968 break;
1969 }
1970 } while (atomic_long_cmpxchg(&ltt_buf->consumed, consumed_old,
1971 consumed_new) != consumed_old);
1972
1973 if (consumed_old != consumed_new) {
1974 /*
1975 * Reader pushed : we are the winner of the push, we can
1976 * therefore reequilibrate reserve and commit. Atomic increment
1977 * of the commit count permits other writers to play around
1978 * with this variable before us. We keep track of
1979 * corrupted_subbuffers even in overwrite mode :
1980 * we never want to write over a non completely committed
1981 * sub-buffer : possible causes : the buffer size is too low
1982 * compared to the unordered data input, or there is a writer
1983 * that died between the reserve and the commit.
1984 */
1985 if (offsets->reserve_commit_diff) {
1986 /*
1987 * We have to alter the sub-buffer commit count.
1988 * We do not deliver the previous subbuffer, given it
1989 * was either corrupted or not consumed (overwrite
1990 * mode).
1991 */
1992 local_add(offsets->reserve_commit_diff,
1993 &ltt_buf->commit_count[
1994 SUBBUF_INDEX(offsets->begin,
1995 buf->chan)]);
1996 if (!ltt_channel->overwrite
1997 || offsets->reserve_commit_diff
1998 != rchan->subbuf_size) {
1999 /*
2000 * The reserve commit diff was not subbuf_size :
2001 * it means the subbuffer was partly written to
2002 * and is therefore corrupted. If it is multiple
2003 * of subbuffer size and we are in flight
2004 * recorder mode, we are skipping over a whole
2005 * subbuffer.
2006 */
2007 local_inc(&ltt_buf->corrupted_subbuffers);
2008 }
2009 }
2010 }
2011 }
2012
2013
2014 /*
2015 * ltt_reserve_switch_old_subbuf: switch old subbuffer
2016 *
2017 * Concurrency safe because we are the last and only thread to alter this
2018 * sub-buffer. As long as it is not delivered and read, no other thread can
2019 * alter the offset, alter the reserve_count or call the
2020 * client_buffer_end_callback on this sub-buffer.
2021 *
2022 * The only remaining threads could be the ones with pending commits. They will
2023 * have to do the deliver themselves. Not concurrency safe in overwrite mode.
2024 * We detect corrupted subbuffers with commit and reserve counts. We keep a
2025 * corrupted sub-buffers count and push the readers across these sub-buffers.
2026 *
2027 * Not concurrency safe if a writer is stalled in a subbuffer and another writer
2028 * switches in, finding out it's corrupted. The result will be than the old
2029 * (uncommited) subbuffer will be declared corrupted, and that the new subbuffer
2030 * will be declared corrupted too because of the commit count adjustment.
2031 *
2032 * Note : offset_old should never be 0 here.
2033 */
2034 static inline void ltt_reserve_switch_old_subbuf(
2035 struct ltt_channel_struct *ltt_channel,
2036 struct ltt_channel_buf_struct *ltt_buf, struct rchan *rchan,
2037 struct rchan_buf *buf,
2038 struct ltt_reserve_switch_offsets *offsets, u64 *tsc)
2039 {
2040 long oldidx = SUBBUF_INDEX(offsets->old - 1, rchan);
2041
2042 ltt_channel->buffer_end(buf, *tsc, offsets->old, oldidx);
2043 /* Must write buffer end before incrementing commit count */
2044 smp_wmb();
2045 offsets->commit_count =
2046 local_add_return(rchan->subbuf_size
2047 - (SUBBUF_OFFSET(offsets->old - 1, rchan)
2048 + 1),
2049 &ltt_buf->commit_count[oldidx]);
2050 if ((BUFFER_TRUNC(offsets->old - 1, rchan)
2051 >> ltt_channel->n_subbufs_order)
2052 - ((offsets->commit_count - rchan->subbuf_size)
2053 & ltt_channel->commit_count_mask) == 0)
2054 ltt_deliver(buf, oldidx, offsets->commit_count);
2055 }
2056
2057 /*
2058 * ltt_reserve_switch_new_subbuf: Populate new subbuffer.
2059 *
2060 * This code can be executed unordered : writers may already have written to the
2061 * sub-buffer before this code gets executed, caution. The commit makes sure
2062 * that this code is executed before the deliver of this sub-buffer.
2063 */
2064 static /*inline*/ void ltt_reserve_switch_new_subbuf(
2065 struct ltt_channel_struct *ltt_channel,
2066 struct ltt_channel_buf_struct *ltt_buf, struct rchan *rchan,
2067 struct rchan_buf *buf,
2068 struct ltt_reserve_switch_offsets *offsets, u64 *tsc)
2069 {
2070 long beginidx = SUBBUF_INDEX(offsets->begin, rchan);
2071
2072 ltt_channel->buffer_begin(buf, *tsc, beginidx);
2073 /* Must write buffer end before incrementing commit count */
2074 smp_wmb();
2075 offsets->commit_count = local_add_return(ltt_subbuffer_header_size(),
2076 &ltt_buf->commit_count[beginidx]);
2077 /* Check if the written buffer has to be delivered */
2078 if ((BUFFER_TRUNC(offsets->begin, rchan)
2079 >> ltt_channel->n_subbufs_order)
2080 - ((offsets->commit_count - rchan->subbuf_size)
2081 & ltt_channel->commit_count_mask) == 0)
2082 ltt_deliver(buf, beginidx, offsets->commit_count);
2083 }
2084
2085
2086 /*
2087 * ltt_reserve_end_switch_current: finish switching current subbuffer
2088 *
2089 * Concurrency safe because we are the last and only thread to alter this
2090 * sub-buffer. As long as it is not delivered and read, no other thread can
2091 * alter the offset, alter the reserve_count or call the
2092 * client_buffer_end_callback on this sub-buffer.
2093 *
2094 * The only remaining threads could be the ones with pending commits. They will
2095 * have to do the deliver themselves. Not concurrency safe in overwrite mode.
2096 * We detect corrupted subbuffers with commit and reserve counts. We keep a
2097 * corrupted sub-buffers count and push the readers across these sub-buffers.
2098 *
2099 * Not concurrency safe if a writer is stalled in a subbuffer and another writer
2100 * switches in, finding out it's corrupted. The result will be than the old
2101 * (uncommited) subbuffer will be declared corrupted, and that the new subbuffer
2102 * will be declared corrupted too because of the commit count adjustment.
2103 */
2104 static inline void ltt_reserve_end_switch_current(
2105 struct ltt_channel_struct *ltt_channel,
2106 struct ltt_channel_buf_struct *ltt_buf, struct rchan *rchan,
2107 struct rchan_buf *buf,
2108 struct ltt_reserve_switch_offsets *offsets, u64 *tsc)
2109 {
2110 long endidx = SUBBUF_INDEX(offsets->end - 1, rchan);
2111
2112 ltt_channel->buffer_end(buf, *tsc, offsets->end, endidx);
2113 /* Must write buffer begin before incrementing commit count */
2114 smp_wmb();
2115 offsets->commit_count =
2116 local_add_return(rchan->subbuf_size
2117 - (SUBBUF_OFFSET(offsets->end - 1, rchan)
2118 + 1),
2119 &ltt_buf->commit_count[endidx]);
2120 if ((BUFFER_TRUNC(offsets->end - 1, rchan)
2121 >> ltt_channel->n_subbufs_order)
2122 - ((offsets->commit_count - rchan->subbuf_size)
2123 & ltt_channel->commit_count_mask) == 0)
2124 ltt_deliver(buf, endidx, offsets->commit_count);
2125 }
2126
2127 /**
2128 * ltt_relay_reserve_slot - Atomic slot reservation in a LTTng buffer.
2129 * @trace: the trace structure to log to.
2130 * @ltt_channel: channel structure
2131 * @transport_data: data structure specific to ltt relay
2132 * @data_size: size of the variable length data to log.
2133 * @slot_size: pointer to total size of the slot (out)
2134 * @buf_offset : pointer to reserved buffer offset (out)
2135 * @tsc: pointer to the tsc at the slot reservation (out)
2136 * @cpu: cpuid
2137 *
2138 * Return : -ENOSPC if not enough space, else returns 0.
2139 * It will take care of sub-buffer switching.
2140 */
2141 static notrace int ltt_relay_reserve_slot(struct ltt_trace_struct *trace,
2142 struct ltt_channel_struct *ltt_channel, void **transport_data,
2143 size_t data_size, size_t *slot_size, long *buf_offset, u64 *tsc,
2144 unsigned int *rflags, int largest_align)
2145 {
2146 struct rchan *rchan = ltt_channel->trans_channel_data;
2147 struct rchan_buf *buf = *transport_data = rchan->buf;
2148 struct ltt_channel_buf_struct *ltt_buf = ltt_channel->buf;
2149 struct ltt_reserve_switch_offsets offsets;
2150
2151 offsets.reserve_commit_diff = 0;
2152 offsets.size = 0;
2153
2154 /*
2155 * Perform retryable operations.
2156 */
2157 if (ltt_nesting > 4) {
2158 local_inc(&ltt_buf->events_lost);
2159 return -EPERM;
2160 }
2161 do {
2162 if (ltt_relay_try_reserve(ltt_channel, ltt_buf,
2163 rchan, buf, &offsets, data_size, tsc, rflags,
2164 largest_align))
2165 return -ENOSPC;
2166 } while (local_cmpxchg(&ltt_buf->offset, offsets.old,
2167 offsets.end) != offsets.old);
2168
2169 /*
2170 * Atomically update last_tsc. This update races against concurrent
2171 * atomic updates, but the race will always cause supplementary full TSC
2172 * events, never the opposite (missing a full TSC event when it would be
2173 * needed).
2174 */
2175 save_last_tsc(ltt_buf, *tsc);
2176
2177 /*
2178 * Push the reader if necessary
2179 */
2180 ltt_reserve_push_reader(ltt_channel, ltt_buf, rchan, buf, &offsets);
2181
2182 /*
2183 * Switch old subbuffer if needed.
2184 */
2185 if (offsets.end_switch_old)
2186 ltt_reserve_switch_old_subbuf(ltt_channel, ltt_buf, rchan, buf,
2187 &offsets, tsc);
2188
2189 /*
2190 * Populate new subbuffer.
2191 */
2192 if (offsets.begin_switch)
2193 ltt_reserve_switch_new_subbuf(ltt_channel, ltt_buf, rchan,
2194 buf, &offsets, tsc);
2195
2196 if (offsets.end_switch_current)
2197 ltt_reserve_end_switch_current(ltt_channel, ltt_buf, rchan,
2198 buf, &offsets, tsc);
2199
2200 *slot_size = offsets.size;
2201 *buf_offset = offsets.begin + offsets.before_hdr_pad;
2202 return 0;
2203 }
2204
2205 /*
2206 * Force a sub-buffer switch for a per-cpu buffer. This operation is
2207 * completely reentrant : can be called while tracing is active with
2208 * absolutely no lock held.
2209 *
2210 * Note, however, that as a local_cmpxchg is used for some atomic
2211 * operations, this function must be called from the CPU which owns the buffer
2212 * for a ACTIVE flush.
2213 */
2214 static notrace void ltt_force_switch(struct rchan_buf *buf,
2215 enum force_switch_mode mode)
2216 {
2217 struct ltt_channel_struct *ltt_channel =
2218 (struct ltt_channel_struct *)buf->chan->private_data;
2219 struct ltt_channel_buf_struct *ltt_buf = ltt_channel->buf;
2220 struct rchan *rchan = ltt_channel->trans_channel_data;
2221 struct ltt_reserve_switch_offsets offsets;
2222 u64 tsc;
2223
2224 offsets.reserve_commit_diff = 0;
2225 offsets.size = 0;
2226
2227 /*
2228 * Perform retryable operations.
2229 */
2230 do {
2231 if (ltt_relay_try_switch(mode, ltt_channel, ltt_buf,
2232 rchan, buf, &offsets, &tsc))
2233 return;
2234 } while (local_cmpxchg(&ltt_buf->offset, offsets.old,
2235 offsets.end) != offsets.old);
2236
2237 /*
2238 * Atomically update last_tsc. This update races against concurrent
2239 * atomic updates, but the race will always cause supplementary full TSC
2240 * events, never the opposite (missing a full TSC event when it would be
2241 * needed).
2242 */
2243 save_last_tsc(ltt_buf, tsc);
2244
2245 /*
2246 * Push the reader if necessary
2247 */
2248 if (mode == FORCE_ACTIVE)
2249 ltt_reserve_push_reader(ltt_channel, ltt_buf, rchan,
2250 buf, &offsets);
2251
2252 /*
2253 * Switch old subbuffer if needed.
2254 */
2255 if (offsets.end_switch_old)
2256 ltt_reserve_switch_old_subbuf(ltt_channel, ltt_buf, rchan, buf,
2257 &offsets, &tsc);
2258
2259 /*
2260 * Populate new subbuffer.
2261 */
2262 if (mode == FORCE_ACTIVE)
2263 ltt_reserve_switch_new_subbuf(ltt_channel,
2264 ltt_buf, rchan, buf, &offsets, &tsc);
2265 }
2266
2267 /*
2268 * This is called with preemption disabled when user space has requested
2269 * blocking mode. If one of the active traces has free space below a
2270 * specific threshold value, we reenable preemption and block.
2271 */
2272 static int ltt_relay_user_blocking(struct ltt_trace_struct *trace,
2273 unsigned int chan_index, size_t data_size,
2274 struct user_dbg_data *dbg)
2275 {
2276 //ust// struct rchan *rchan;
2277 //ust// struct ltt_channel_buf_struct *ltt_buf;
2278 //ust// struct ltt_channel_struct *channel;
2279 //ust// struct rchan_buf *relay_buf;
2280 //ust// int cpu;
2281 //ust// DECLARE_WAITQUEUE(wait, current);
2282 //ust//
2283 //ust// channel = &trace->channels[chan_index];
2284 //ust// rchan = channel->trans_channel_data;
2285 //ust// cpu = smp_processor_id();
2286 //ust// relay_buf = rchan->buf[cpu];
2287 //ust// ltt_buf = percpu_ptr(channel->buf, cpu);
2288 //ust//
2289 //ust// /*
2290 //ust// * Check if data is too big for the channel : do not
2291 //ust// * block for it.
2292 //ust// */
2293 //ust// if (LTT_RESERVE_CRITICAL + data_size > relay_buf->chan->subbuf_size)
2294 //ust// return 0;
2295 //ust//
2296 //ust// /*
2297 //ust// * If free space too low, we block. We restart from the
2298 //ust// * beginning after we resume (cpu id may have changed
2299 //ust// * while preemption is active).
2300 //ust// */
2301 //ust// spin_lock(&ltt_buf->full_lock);
2302 //ust// if (!channel->overwrite) {
2303 //ust// dbg->write = local_read(&ltt_buf->offset);
2304 //ust// dbg->read = atomic_long_read(&ltt_buf->consumed);
2305 //ust// dbg->avail_size = dbg->write + LTT_RESERVE_CRITICAL + data_size
2306 //ust// - SUBBUF_TRUNC(dbg->read,
2307 //ust// relay_buf->chan);
2308 //ust// if (dbg->avail_size > rchan->alloc_size) {
2309 //ust// __set_current_state(TASK_INTERRUPTIBLE);
2310 //ust// add_wait_queue(&ltt_buf->write_wait, &wait);
2311 //ust// spin_unlock(&ltt_buf->full_lock);
2312 //ust// preempt_enable();
2313 //ust// schedule();
2314 //ust// __set_current_state(TASK_RUNNING);
2315 //ust// remove_wait_queue(&ltt_buf->write_wait, &wait);
2316 //ust// if (signal_pending(current))
2317 //ust// return -ERESTARTSYS;
2318 //ust// preempt_disable();
2319 //ust// return 1;
2320 //ust// }
2321 //ust// }
2322 //ust// spin_unlock(&ltt_buf->full_lock);
2323 return 0;
2324 }
2325
2326 static void ltt_relay_print_user_errors(struct ltt_trace_struct *trace,
2327 unsigned int chan_index, size_t data_size,
2328 struct user_dbg_data *dbg)
2329 {
2330 struct rchan *rchan;
2331 struct ltt_channel_buf_struct *ltt_buf;
2332 struct ltt_channel_struct *channel;
2333 struct rchan_buf *relay_buf;
2334
2335 channel = &trace->channels[chan_index];
2336 rchan = channel->trans_channel_data;
2337 relay_buf = rchan->buf;
2338 ltt_buf = channel->buf;
2339
2340 printk(KERN_ERR "Error in LTT usertrace : "
2341 "buffer full : event lost in blocking "
2342 "mode. Increase LTT_RESERVE_CRITICAL.\n");
2343 printk(KERN_ERR "LTT nesting level is %u.\n", ltt_nesting);
2344 printk(KERN_ERR "LTT avail size %lu.\n",
2345 dbg->avail_size);
2346 printk(KERN_ERR "avai write : %lu, read : %lu\n",
2347 dbg->write, dbg->read);
2348
2349 dbg->write = local_read(&ltt_buf->offset);
2350 dbg->read = atomic_long_read(&ltt_buf->consumed);
2351
2352 printk(KERN_ERR "LTT cur size %lu.\n",
2353 dbg->write + LTT_RESERVE_CRITICAL + data_size
2354 - SUBBUF_TRUNC(dbg->read, relay_buf->chan));
2355 printk(KERN_ERR "cur write : %lu, read : %lu\n",
2356 dbg->write, dbg->read);
2357 }
2358
2359 //ust// static struct ltt_transport ltt_relay_transport = {
2360 //ust// .name = "relay",
2361 //ust// .owner = THIS_MODULE,
2362 //ust// .ops = {
2363 //ust// .create_dirs = ltt_relay_create_dirs,
2364 //ust// .remove_dirs = ltt_relay_remove_dirs,
2365 //ust// .create_channel = ltt_relay_create_channel,
2366 //ust// .finish_channel = ltt_relay_finish_channel,
2367 //ust// .remove_channel = ltt_relay_remove_channel,
2368 //ust// .wakeup_channel = ltt_relay_async_wakeup_chan,
2369 //ust// .commit_slot = ltt_relay_commit_slot,
2370 //ust// .reserve_slot = ltt_relay_reserve_slot,
2371 //ust// .user_blocking = ltt_relay_user_blocking,
2372 //ust// .user_errors = ltt_relay_print_user_errors,
2373 //ust// },
2374 //ust// };
2375
2376 static struct ltt_transport ust_relay_transport = {
2377 .name = "ustrelay",
2378 //ust// .owner = THIS_MODULE,
2379 .ops = {
2380 .create_dirs = ltt_relay_create_dirs,
2381 .remove_dirs = ltt_relay_remove_dirs,
2382 .create_channel = ltt_relay_create_channel,
2383 .finish_channel = ltt_relay_finish_channel,
2384 .remove_channel = ltt_relay_remove_channel,
2385 .wakeup_channel = ltt_relay_async_wakeup_chan,
2386 // .commit_slot = ltt_relay_commit_slot,
2387 .reserve_slot = ltt_relay_reserve_slot,
2388 .user_blocking = ltt_relay_user_blocking,
2389 .user_errors = ltt_relay_print_user_errors,
2390 },
2391 };
2392
2393 //ust// static int __init ltt_relay_init(void)
2394 //ust// {
2395 //ust// printk(KERN_INFO "LTT : ltt-relay init\n");
2396 //ust//
2397 //ust// ltt_file_operations = ltt_relay_file_operations;
2398 //ust// ltt_file_operations.owner = THIS_MODULE;
2399 //ust// ltt_file_operations.open = ltt_open;
2400 //ust// ltt_file_operations.release = ltt_release;
2401 //ust// ltt_file_operations.poll = ltt_poll;
2402 //ust// ltt_file_operations.splice_read = ltt_relay_file_splice_read,
2403 //ust// ltt_file_operations.ioctl = ltt_ioctl;
2404 //ust//#ifdef CONFIG_COMPAT
2405 //ust// ltt_file_operations.compat_ioctl = ltt_compat_ioctl;
2406 //ust//#endif
2407 //ust//
2408 //ust// ltt_transport_register(&ltt_relay_transport);
2409 //ust//
2410 //ust// return 0;
2411 //ust// }
2412
2413 /*
2414 * for flight recording. must be called after relay_commit.
2415 * This function decrements de subbuffer's lost_size each time the commit count
2416 * reaches back the reserve offset (module subbuffer size). It is useful for
2417 * crash dump.
2418 */
2419 //ust// #ifdef CONFIG_LTT_VMCORE
2420 static /* inline */ void ltt_write_commit_counter(struct rchan_buf *buf,
2421 struct ltt_channel_buf_struct *ltt_buf,
2422 long idx, long buf_offset, long commit_count, size_t data_size)
2423 {
2424 long offset;
2425 long commit_seq_old;
2426
2427 offset = buf_offset + data_size;
2428
2429 /*
2430 * SUBBUF_OFFSET includes commit_count_mask. We can simply
2431 * compare the offsets within the subbuffer without caring about
2432 * buffer full/empty mismatch because offset is never zero here
2433 * (subbuffer header and event headers have non-zero length).
2434 */
2435 if (unlikely(SUBBUF_OFFSET(offset - commit_count, buf->chan)))
2436 return;
2437
2438 commit_seq_old = local_read(&ltt_buf->commit_seq[idx]);
2439 while (commit_seq_old < commit_count)
2440 commit_seq_old = local_cmpxchg(&ltt_buf->commit_seq[idx],
2441 commit_seq_old, commit_count);
2442 }
2443 //ust// #else
2444 //ust// static inline void ltt_write_commit_counter(struct rchan_buf *buf,
2445 //ust// long buf_offset, size_t slot_size)
2446 //ust// {
2447 //ust// }
2448 //ust// #endif
2449
2450 /*
2451 * Atomic unordered slot commit. Increments the commit count in the
2452 * specified sub-buffer, and delivers it if necessary.
2453 *
2454 * Parameters:
2455 *
2456 * @ltt_channel : channel structure
2457 * @transport_data: transport-specific data
2458 * @buf_offset : offset following the event header.
2459 * @data_size : size of the event data.
2460 * @slot_size : size of the reserved slot.
2461 */
2462 /* FIXME: make this function static inline in the .h! */
2463 /*static*/ /* inline */ notrace void ltt_commit_slot(
2464 struct ltt_channel_struct *ltt_channel,
2465 void **transport_data, long buf_offset,
2466 size_t data_size, size_t slot_size)
2467 {
2468 struct rchan_buf *buf = *transport_data;
2469 struct ltt_channel_buf_struct *ltt_buf = ltt_channel->buf;
2470 struct rchan *rchan = buf->chan;
2471 long offset_end = buf_offset;
2472 long endidx = SUBBUF_INDEX(offset_end - 1, rchan);
2473 long commit_count;
2474
2475 /* Must write slot data before incrementing commit count */
2476 smp_wmb();
2477 commit_count = local_add_return(slot_size,
2478 &ltt_buf->commit_count[endidx]);
2479 /* Check if all commits have been done */
2480 if ((BUFFER_TRUNC(offset_end - 1, rchan)
2481 >> ltt_channel->n_subbufs_order)
2482 - ((commit_count - rchan->subbuf_size)
2483 & ltt_channel->commit_count_mask) == 0)
2484 ltt_deliver(buf, endidx, commit_count);
2485 /*
2486 * Update lost_size for each commit. It's needed only for extracting
2487 * ltt buffers from vmcore, after crash.
2488 */
2489 ltt_write_commit_counter(buf, ltt_buf, endidx,
2490 buf_offset, commit_count, data_size);
2491 }
2492
2493
2494 static char initialized = 0;
2495
2496 void __attribute__((constructor)) init_ustrelay_transport(void)
2497 {
2498 if(!initialized) {
2499 ltt_transport_register(&ust_relay_transport);
2500 initialized = 1;
2501 }
2502 }
2503
2504 static void __attribute__((destructor)) ltt_relay_exit(void)
2505 {
2506 //ust// printk(KERN_INFO "LTT : ltt-relay exit\n");
2507
2508 ltt_transport_unregister(&ust_relay_transport);
2509 }
2510
2511 //ust// module_init(ltt_relay_init);
2512 //ust// module_exit(ltt_relay_exit);
2513 //ust//
2514 //ust// MODULE_LICENSE("GPL");
2515 //ust// MODULE_AUTHOR("Mathieu Desnoyers");
2516 //ust// MODULE_DESCRIPTION("Linux Trace Toolkit Next Generation Lockless Relay");
This page took 0.074687 seconds and 3 git commands to generate.