Commit | Line | Data |
---|---|---|
f3bc08c5 MD |
1 | /* |
2 | * ring_buffer_iterator.c | |
3 | * | |
f3bc08c5 MD |
4 | * Ring buffer and channel iterators. Get each event of a channel in order. Uses |
5 | * a prio heap for per-cpu buffers, giving a O(log(NR_CPUS)) algorithmic | |
6 | * complexity for the "get next event" operation. | |
7 | * | |
886d51a3 MD |
8 | * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
9 | * | |
10 | * This library is free software; you can redistribute it and/or | |
11 | * modify it under the terms of the GNU Lesser General Public | |
12 | * License as published by the Free Software Foundation; only | |
13 | * version 2.1 of the License. | |
14 | * | |
15 | * This library is distributed in the hope that it will be useful, | |
16 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
18 | * Lesser General Public License for more details. | |
19 | * | |
20 | * You should have received a copy of the GNU Lesser General Public | |
21 | * License along with this library; if not, write to the Free Software | |
22 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
23 | * | |
f3bc08c5 MD |
24 | * Author: |
25 | * Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
f3bc08c5 MD |
26 | */ |
27 | ||
28 | #include "../../wrapper/ringbuffer/iterator.h" | |
29 | #include <linux/jiffies.h> | |
30 | #include <linux/delay.h> | |
31 | #include <linux/module.h> | |
32 | ||
33 | /* | |
34 | * Safety factor taking into account internal kernel interrupt latency. | |
35 | * Assuming 250ms worse-case latency. | |
36 | */ | |
37 | #define MAX_SYSTEM_LATENCY 250 | |
38 | ||
39 | /* | |
40 | * Maximum delta expected between trace clocks. At most 1 jiffy delta. | |
41 | */ | |
42 | #define MAX_CLOCK_DELTA (jiffies_to_usecs(1) * 1000) | |
43 | ||
44 | /** | |
45 | * lib_ring_buffer_get_next_record - Get the next record in a buffer. | |
46 | * @chan: channel | |
47 | * @buf: buffer | |
48 | * | |
49 | * Returns the size of the event read, -EAGAIN if buffer is empty, -ENODATA if | |
50 | * buffer is empty and finalized. The buffer must already be opened for reading. | |
51 | */ | |
52 | ssize_t lib_ring_buffer_get_next_record(struct channel *chan, | |
53 | struct lib_ring_buffer *buf) | |
54 | { | |
5a8fd222 | 55 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
56 | struct lib_ring_buffer_iter *iter = &buf->iter; |
57 | int ret; | |
58 | ||
59 | restart: | |
60 | switch (iter->state) { | |
61 | case ITER_GET_SUBBUF: | |
62 | ret = lib_ring_buffer_get_next_subbuf(buf); | |
63 | if (ret && !ACCESS_ONCE(buf->finalized) | |
64 | && config->alloc == RING_BUFFER_ALLOC_GLOBAL) { | |
65 | /* | |
66 | * Use "pull" scheme for global buffers. The reader | |
67 | * itself flushes the buffer to "pull" data not visible | |
68 | * to readers yet. Flush current subbuffer and re-try. | |
69 | * | |
70 | * Per-CPU buffers rather use a "push" scheme because | |
71 | * the IPI needed to flush all CPU's buffers is too | |
72 | * costly. In the "push" scheme, the reader waits for | |
73 | * the writer periodic deferrable timer to flush the | |
74 | * buffers (keeping track of a quiescent state | |
75 | * timestamp). Therefore, the writer "pushes" data out | |
76 | * of the buffers rather than letting the reader "pull" | |
77 | * data from the buffer. | |
78 | */ | |
79 | lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE); | |
80 | ret = lib_ring_buffer_get_next_subbuf(buf); | |
81 | } | |
82 | if (ret) | |
83 | return ret; | |
84 | iter->consumed = buf->cons_snapshot; | |
85 | iter->data_size = lib_ring_buffer_get_read_data_size(config, buf); | |
86 | iter->read_offset = iter->consumed; | |
87 | /* skip header */ | |
88 | iter->read_offset += config->cb.subbuffer_header_size(); | |
89 | iter->state = ITER_TEST_RECORD; | |
90 | goto restart; | |
91 | case ITER_TEST_RECORD: | |
92 | if (iter->read_offset - iter->consumed >= iter->data_size) { | |
93 | iter->state = ITER_PUT_SUBBUF; | |
94 | } else { | |
95 | CHAN_WARN_ON(chan, !config->cb.record_get); | |
96 | config->cb.record_get(config, chan, buf, | |
97 | iter->read_offset, | |
98 | &iter->header_len, | |
99 | &iter->payload_len, | |
100 | &iter->timestamp); | |
101 | iter->read_offset += iter->header_len; | |
102 | subbuffer_consume_record(config, &buf->backend); | |
103 | iter->state = ITER_NEXT_RECORD; | |
104 | return iter->payload_len; | |
105 | } | |
106 | goto restart; | |
107 | case ITER_NEXT_RECORD: | |
108 | iter->read_offset += iter->payload_len; | |
109 | iter->state = ITER_TEST_RECORD; | |
110 | goto restart; | |
111 | case ITER_PUT_SUBBUF: | |
112 | lib_ring_buffer_put_next_subbuf(buf); | |
113 | iter->state = ITER_GET_SUBBUF; | |
114 | goto restart; | |
115 | default: | |
116 | CHAN_WARN_ON(chan, 1); /* Should not happen */ | |
117 | return -EPERM; | |
118 | } | |
119 | } | |
120 | EXPORT_SYMBOL_GPL(lib_ring_buffer_get_next_record); | |
121 | ||
122 | static int buf_is_higher(void *a, void *b) | |
123 | { | |
124 | struct lib_ring_buffer *bufa = a; | |
125 | struct lib_ring_buffer *bufb = b; | |
126 | ||
127 | /* Consider lowest timestamps to be at the top of the heap */ | |
128 | return (bufa->iter.timestamp < bufb->iter.timestamp); | |
129 | } | |
130 | ||
131 | static | |
132 | void lib_ring_buffer_get_empty_buf_records(const struct lib_ring_buffer_config *config, | |
133 | struct channel *chan) | |
134 | { | |
a88db018 | 135 | struct lttng_ptr_heap *heap = &chan->iter.heap; |
f3bc08c5 MD |
136 | struct lib_ring_buffer *buf, *tmp; |
137 | ssize_t len; | |
138 | ||
139 | list_for_each_entry_safe(buf, tmp, &chan->iter.empty_head, | |
140 | iter.empty_node) { | |
141 | len = lib_ring_buffer_get_next_record(chan, buf); | |
142 | ||
143 | /* | |
144 | * Deal with -EAGAIN and -ENODATA. | |
145 | * len >= 0 means record contains data. | |
146 | * -EBUSY should never happen, because we support only one | |
147 | * reader. | |
148 | */ | |
149 | switch (len) { | |
150 | case -EAGAIN: | |
151 | /* Keep node in empty list */ | |
152 | break; | |
153 | case -ENODATA: | |
154 | /* | |
155 | * Buffer is finalized. Don't add to list of empty | |
156 | * buffer, because it has no more data to provide, ever. | |
157 | */ | |
158 | list_del(&buf->iter.empty_node); | |
159 | break; | |
160 | case -EBUSY: | |
161 | CHAN_WARN_ON(chan, 1); | |
162 | break; | |
163 | default: | |
164 | /* | |
165 | * Insert buffer into the heap, remove from empty buffer | |
ab2277d6 | 166 | * list. |
f3bc08c5 MD |
167 | */ |
168 | CHAN_WARN_ON(chan, len < 0); | |
169 | list_del(&buf->iter.empty_node); | |
a88db018 | 170 | CHAN_WARN_ON(chan, lttng_heap_insert(heap, buf)); |
f3bc08c5 MD |
171 | } |
172 | } | |
173 | } | |
174 | ||
175 | static | |
176 | void lib_ring_buffer_wait_for_qs(const struct lib_ring_buffer_config *config, | |
177 | struct channel *chan) | |
178 | { | |
179 | u64 timestamp_qs; | |
180 | unsigned long wait_msecs; | |
181 | ||
182 | /* | |
183 | * No need to wait if no empty buffers are present. | |
184 | */ | |
185 | if (list_empty(&chan->iter.empty_head)) | |
186 | return; | |
187 | ||
188 | timestamp_qs = config->cb.ring_buffer_clock_read(chan); | |
189 | /* | |
190 | * We need to consider previously empty buffers. | |
191 | * Do a get next buf record on each of them. Add them to | |
192 | * the heap if they have data. If at least one of them | |
193 | * don't have data, we need to wait for | |
194 | * switch_timer_interval + MAX_SYSTEM_LATENCY (so we are sure the | |
195 | * buffers have been switched either by the timer or idle entry) and | |
196 | * check them again, adding them if they have data. | |
197 | */ | |
198 | lib_ring_buffer_get_empty_buf_records(config, chan); | |
199 | ||
200 | /* | |
201 | * No need to wait if no empty buffers are present. | |
202 | */ | |
203 | if (list_empty(&chan->iter.empty_head)) | |
204 | return; | |
205 | ||
206 | /* | |
207 | * We need to wait for the buffer switch timer to run. If the | |
208 | * CPU is idle, idle entry performed the switch. | |
209 | * TODO: we could optimize further by skipping the sleep if all | |
210 | * empty buffers belong to idle or offline cpus. | |
211 | */ | |
212 | wait_msecs = jiffies_to_msecs(chan->switch_timer_interval); | |
213 | wait_msecs += MAX_SYSTEM_LATENCY; | |
214 | msleep(wait_msecs); | |
215 | lib_ring_buffer_get_empty_buf_records(config, chan); | |
216 | /* | |
217 | * Any buffer still in the empty list here cannot possibly | |
218 | * contain an event with a timestamp prior to "timestamp_qs". | |
219 | * The new quiescent state timestamp is the one we grabbed | |
220 | * before waiting for buffer data. It is therefore safe to | |
221 | * ignore empty buffers up to last_qs timestamp for fusion | |
222 | * merge. | |
223 | */ | |
224 | chan->iter.last_qs = timestamp_qs; | |
225 | } | |
226 | ||
227 | /** | |
228 | * channel_get_next_record - Get the next record in a channel. | |
229 | * @chan: channel | |
230 | * @ret_buf: the buffer in which the event is located (output) | |
231 | * | |
232 | * Returns the size of new current event, -EAGAIN if all buffers are empty, | |
233 | * -ENODATA if all buffers are empty and finalized. The channel must already be | |
234 | * opened for reading. | |
235 | */ | |
236 | ||
237 | ssize_t channel_get_next_record(struct channel *chan, | |
238 | struct lib_ring_buffer **ret_buf) | |
239 | { | |
5a8fd222 | 240 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 | 241 | struct lib_ring_buffer *buf; |
a88db018 | 242 | struct lttng_ptr_heap *heap; |
f3bc08c5 MD |
243 | ssize_t len; |
244 | ||
245 | if (config->alloc == RING_BUFFER_ALLOC_GLOBAL) { | |
246 | *ret_buf = channel_get_ring_buffer(config, chan, 0); | |
247 | return lib_ring_buffer_get_next_record(chan, *ret_buf); | |
248 | } | |
249 | ||
250 | heap = &chan->iter.heap; | |
251 | ||
252 | /* | |
253 | * get next record for topmost buffer. | |
254 | */ | |
a88db018 | 255 | buf = lttng_heap_maximum(heap); |
f3bc08c5 MD |
256 | if (buf) { |
257 | len = lib_ring_buffer_get_next_record(chan, buf); | |
258 | /* | |
259 | * Deal with -EAGAIN and -ENODATA. | |
260 | * len >= 0 means record contains data. | |
261 | */ | |
262 | switch (len) { | |
263 | case -EAGAIN: | |
264 | buf->iter.timestamp = 0; | |
265 | list_add(&buf->iter.empty_node, &chan->iter.empty_head); | |
266 | /* Remove topmost buffer from the heap */ | |
a88db018 | 267 | CHAN_WARN_ON(chan, lttng_heap_remove(heap) != buf); |
f3bc08c5 MD |
268 | break; |
269 | case -ENODATA: | |
270 | /* | |
271 | * Buffer is finalized. Remove buffer from heap and | |
272 | * don't add to list of empty buffer, because it has no | |
273 | * more data to provide, ever. | |
274 | */ | |
a88db018 | 275 | CHAN_WARN_ON(chan, lttng_heap_remove(heap) != buf); |
f3bc08c5 MD |
276 | break; |
277 | case -EBUSY: | |
278 | CHAN_WARN_ON(chan, 1); | |
279 | break; | |
280 | default: | |
281 | /* | |
282 | * Reinsert buffer into the heap. Note that heap can be | |
283 | * partially empty, so we need to use | |
a88db018 | 284 | * lttng_heap_replace_max(). |
f3bc08c5 MD |
285 | */ |
286 | CHAN_WARN_ON(chan, len < 0); | |
a88db018 | 287 | CHAN_WARN_ON(chan, lttng_heap_replace_max(heap, buf) != buf); |
f3bc08c5 MD |
288 | break; |
289 | } | |
290 | } | |
291 | ||
a88db018 | 292 | buf = lttng_heap_maximum(heap); |
f3bc08c5 MD |
293 | if (!buf || buf->iter.timestamp > chan->iter.last_qs) { |
294 | /* | |
295 | * Deal with buffers previously showing no data. | |
296 | * Add buffers containing data to the heap, update | |
297 | * last_qs. | |
298 | */ | |
299 | lib_ring_buffer_wait_for_qs(config, chan); | |
300 | } | |
301 | ||
a88db018 | 302 | *ret_buf = buf = lttng_heap_maximum(heap); |
f3bc08c5 MD |
303 | if (buf) { |
304 | /* | |
305 | * If this warning triggers, you probably need to check your | |
306 | * system interrupt latency. Typical causes: too many printk() | |
307 | * output going to a serial console with interrupts off. | |
308 | * Allow for MAX_CLOCK_DELTA ns timestamp delta going backward. | |
309 | * Observed on SMP KVM setups with trace_clock(). | |
310 | */ | |
311 | if (chan->iter.last_timestamp | |
312 | > (buf->iter.timestamp + MAX_CLOCK_DELTA)) { | |
313 | printk(KERN_WARNING "ring_buffer: timestamps going " | |
314 | "backward. Last time %llu ns, cpu %d, " | |
315 | "current time %llu ns, cpu %d, " | |
316 | "delta %llu ns.\n", | |
317 | chan->iter.last_timestamp, chan->iter.last_cpu, | |
318 | buf->iter.timestamp, buf->backend.cpu, | |
319 | chan->iter.last_timestamp - buf->iter.timestamp); | |
320 | CHAN_WARN_ON(chan, 1); | |
321 | } | |
322 | chan->iter.last_timestamp = buf->iter.timestamp; | |
323 | chan->iter.last_cpu = buf->backend.cpu; | |
324 | return buf->iter.payload_len; | |
325 | } else { | |
326 | /* Heap is empty */ | |
327 | if (list_empty(&chan->iter.empty_head)) | |
328 | return -ENODATA; /* All buffers finalized */ | |
329 | else | |
330 | return -EAGAIN; /* Temporarily empty */ | |
331 | } | |
332 | } | |
333 | EXPORT_SYMBOL_GPL(channel_get_next_record); | |
334 | ||
335 | static | |
336 | void lib_ring_buffer_iterator_init(struct channel *chan, struct lib_ring_buffer *buf) | |
337 | { | |
338 | if (buf->iter.allocated) | |
339 | return; | |
340 | ||
341 | buf->iter.allocated = 1; | |
342 | if (chan->iter.read_open && !buf->iter.read_open) { | |
343 | CHAN_WARN_ON(chan, lib_ring_buffer_open_read(buf) != 0); | |
344 | buf->iter.read_open = 1; | |
345 | } | |
346 | ||
347 | /* Add to list of buffers without any current record */ | |
5a8fd222 | 348 | if (chan->backend.config.alloc == RING_BUFFER_ALLOC_PER_CPU) |
f3bc08c5 MD |
349 | list_add(&buf->iter.empty_node, &chan->iter.empty_head); |
350 | } | |
351 | ||
352 | #ifdef CONFIG_HOTPLUG_CPU | |
353 | static | |
354 | int __cpuinit channel_iterator_cpu_hotplug(struct notifier_block *nb, | |
355 | unsigned long action, | |
356 | void *hcpu) | |
357 | { | |
358 | unsigned int cpu = (unsigned long)hcpu; | |
359 | struct channel *chan = container_of(nb, struct channel, | |
360 | hp_iter_notifier); | |
361 | struct lib_ring_buffer *buf = per_cpu_ptr(chan->backend.buf, cpu); | |
5a8fd222 | 362 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
363 | |
364 | if (!chan->hp_iter_enable) | |
365 | return NOTIFY_DONE; | |
366 | ||
367 | CHAN_WARN_ON(chan, config->alloc == RING_BUFFER_ALLOC_GLOBAL); | |
368 | ||
369 | switch (action) { | |
370 | case CPU_DOWN_FAILED: | |
371 | case CPU_DOWN_FAILED_FROZEN: | |
372 | case CPU_ONLINE: | |
373 | case CPU_ONLINE_FROZEN: | |
374 | lib_ring_buffer_iterator_init(chan, buf); | |
375 | return NOTIFY_OK; | |
376 | default: | |
377 | return NOTIFY_DONE; | |
378 | } | |
379 | } | |
380 | #endif | |
381 | ||
382 | int channel_iterator_init(struct channel *chan) | |
383 | { | |
5a8fd222 | 384 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
385 | struct lib_ring_buffer *buf; |
386 | ||
387 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { | |
388 | int cpu, ret; | |
389 | ||
390 | INIT_LIST_HEAD(&chan->iter.empty_head); | |
a88db018 | 391 | ret = lttng_heap_init(&chan->iter.heap, |
41affe31 | 392 | num_possible_cpus(), |
f3bc08c5 MD |
393 | GFP_KERNEL, buf_is_higher); |
394 | if (ret) | |
395 | return ret; | |
396 | /* | |
397 | * In case of non-hotplug cpu, if the ring-buffer is allocated | |
398 | * in early initcall, it will not be notified of secondary cpus. | |
399 | * In that off case, we need to allocate for all possible cpus. | |
400 | */ | |
401 | #ifdef CONFIG_HOTPLUG_CPU | |
402 | chan->hp_iter_notifier.notifier_call = | |
403 | channel_iterator_cpu_hotplug; | |
404 | chan->hp_iter_notifier.priority = 10; | |
405 | register_cpu_notifier(&chan->hp_iter_notifier); | |
406 | get_online_cpus(); | |
407 | for_each_online_cpu(cpu) { | |
408 | buf = per_cpu_ptr(chan->backend.buf, cpu); | |
409 | lib_ring_buffer_iterator_init(chan, buf); | |
410 | } | |
411 | chan->hp_iter_enable = 1; | |
412 | put_online_cpus(); | |
413 | #else | |
414 | for_each_possible_cpu(cpu) { | |
415 | buf = per_cpu_ptr(chan->backend.buf, cpu); | |
416 | lib_ring_buffer_iterator_init(chan, buf); | |
417 | } | |
418 | #endif | |
419 | } else { | |
420 | buf = channel_get_ring_buffer(config, chan, 0); | |
421 | lib_ring_buffer_iterator_init(chan, buf); | |
422 | } | |
423 | return 0; | |
424 | } | |
425 | ||
426 | void channel_iterator_unregister_notifiers(struct channel *chan) | |
427 | { | |
5a8fd222 | 428 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
429 | |
430 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { | |
431 | chan->hp_iter_enable = 0; | |
432 | unregister_cpu_notifier(&chan->hp_iter_notifier); | |
433 | } | |
434 | } | |
435 | ||
436 | void channel_iterator_free(struct channel *chan) | |
437 | { | |
5a8fd222 | 438 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
439 | |
440 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) | |
a88db018 | 441 | lttng_heap_free(&chan->iter.heap); |
f3bc08c5 MD |
442 | } |
443 | ||
444 | int lib_ring_buffer_iterator_open(struct lib_ring_buffer *buf) | |
445 | { | |
446 | struct channel *chan = buf->backend.chan; | |
5a8fd222 | 447 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
448 | CHAN_WARN_ON(chan, config->output != RING_BUFFER_ITERATOR); |
449 | return lib_ring_buffer_open_read(buf); | |
450 | } | |
451 | EXPORT_SYMBOL_GPL(lib_ring_buffer_iterator_open); | |
452 | ||
453 | /* | |
454 | * Note: Iterators must not be mixed with other types of outputs, because an | |
455 | * iterator can leave the buffer in "GET" state, which is not consistent with | |
456 | * other types of output (mmap, splice, raw data read). | |
457 | */ | |
458 | void lib_ring_buffer_iterator_release(struct lib_ring_buffer *buf) | |
459 | { | |
460 | lib_ring_buffer_release_read(buf); | |
461 | } | |
462 | EXPORT_SYMBOL_GPL(lib_ring_buffer_iterator_release); | |
463 | ||
464 | int channel_iterator_open(struct channel *chan) | |
465 | { | |
5a8fd222 | 466 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
467 | struct lib_ring_buffer *buf; |
468 | int ret = 0, cpu; | |
469 | ||
470 | CHAN_WARN_ON(chan, config->output != RING_BUFFER_ITERATOR); | |
471 | ||
472 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { | |
473 | get_online_cpus(); | |
474 | /* Allow CPU hotplug to keep track of opened reader */ | |
475 | chan->iter.read_open = 1; | |
476 | for_each_channel_cpu(cpu, chan) { | |
477 | buf = channel_get_ring_buffer(config, chan, cpu); | |
478 | ret = lib_ring_buffer_iterator_open(buf); | |
479 | if (ret) | |
480 | goto error; | |
481 | buf->iter.read_open = 1; | |
482 | } | |
483 | put_online_cpus(); | |
484 | } else { | |
485 | buf = channel_get_ring_buffer(config, chan, 0); | |
486 | ret = lib_ring_buffer_iterator_open(buf); | |
487 | } | |
488 | return ret; | |
489 | error: | |
490 | /* Error should always happen on CPU 0, hence no close is required. */ | |
491 | CHAN_WARN_ON(chan, cpu != 0); | |
492 | put_online_cpus(); | |
493 | return ret; | |
494 | } | |
495 | EXPORT_SYMBOL_GPL(channel_iterator_open); | |
496 | ||
497 | void channel_iterator_release(struct channel *chan) | |
498 | { | |
5a8fd222 | 499 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
500 | struct lib_ring_buffer *buf; |
501 | int cpu; | |
502 | ||
503 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { | |
504 | get_online_cpus(); | |
505 | for_each_channel_cpu(cpu, chan) { | |
506 | buf = channel_get_ring_buffer(config, chan, cpu); | |
507 | if (buf->iter.read_open) { | |
508 | lib_ring_buffer_iterator_release(buf); | |
509 | buf->iter.read_open = 0; | |
510 | } | |
511 | } | |
512 | chan->iter.read_open = 0; | |
513 | put_online_cpus(); | |
514 | } else { | |
515 | buf = channel_get_ring_buffer(config, chan, 0); | |
516 | lib_ring_buffer_iterator_release(buf); | |
517 | } | |
518 | } | |
519 | EXPORT_SYMBOL_GPL(channel_iterator_release); | |
520 | ||
521 | void lib_ring_buffer_iterator_reset(struct lib_ring_buffer *buf) | |
522 | { | |
523 | struct channel *chan = buf->backend.chan; | |
524 | ||
525 | if (buf->iter.state != ITER_GET_SUBBUF) | |
526 | lib_ring_buffer_put_next_subbuf(buf); | |
527 | buf->iter.state = ITER_GET_SUBBUF; | |
528 | /* Remove from heap (if present). */ | |
a88db018 | 529 | if (lttng_heap_cherrypick(&chan->iter.heap, buf)) |
f3bc08c5 MD |
530 | list_add(&buf->iter.empty_node, &chan->iter.empty_head); |
531 | buf->iter.timestamp = 0; | |
532 | buf->iter.header_len = 0; | |
533 | buf->iter.payload_len = 0; | |
534 | buf->iter.consumed = 0; | |
535 | buf->iter.read_offset = 0; | |
536 | buf->iter.data_size = 0; | |
537 | /* Don't reset allocated and read_open */ | |
538 | } | |
539 | ||
540 | void channel_iterator_reset(struct channel *chan) | |
541 | { | |
5a8fd222 | 542 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
543 | struct lib_ring_buffer *buf; |
544 | int cpu; | |
545 | ||
546 | /* Empty heap, put into empty_head */ | |
a88db018 | 547 | while ((buf = lttng_heap_remove(&chan->iter.heap)) != NULL) |
f3bc08c5 MD |
548 | list_add(&buf->iter.empty_node, &chan->iter.empty_head); |
549 | ||
550 | for_each_channel_cpu(cpu, chan) { | |
551 | buf = channel_get_ring_buffer(config, chan, cpu); | |
552 | lib_ring_buffer_iterator_reset(buf); | |
553 | } | |
554 | /* Don't reset read_open */ | |
555 | chan->iter.last_qs = 0; | |
556 | chan->iter.last_timestamp = 0; | |
557 | chan->iter.last_cpu = 0; | |
558 | chan->iter.len_left = 0; | |
559 | } | |
560 | ||
561 | /* | |
562 | * Ring buffer payload extraction read() implementation. | |
563 | */ | |
564 | static | |
565 | ssize_t channel_ring_buffer_file_read(struct file *filp, | |
566 | char __user *user_buf, | |
567 | size_t count, | |
568 | loff_t *ppos, | |
569 | struct channel *chan, | |
570 | struct lib_ring_buffer *buf, | |
571 | int fusionmerge) | |
572 | { | |
5a8fd222 | 573 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
574 | size_t read_count = 0, read_offset; |
575 | ssize_t len; | |
576 | ||
577 | might_sleep(); | |
578 | if (!access_ok(VERIFY_WRITE, user_buf, count)) | |
579 | return -EFAULT; | |
580 | ||
581 | /* Finish copy of previous record */ | |
582 | if (*ppos != 0) { | |
583 | if (read_count < count) { | |
584 | len = chan->iter.len_left; | |
585 | read_offset = *ppos; | |
586 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU | |
587 | && fusionmerge) | |
a88db018 | 588 | buf = lttng_heap_maximum(&chan->iter.heap); |
f3bc08c5 MD |
589 | CHAN_WARN_ON(chan, !buf); |
590 | goto skip_get_next; | |
591 | } | |
592 | } | |
593 | ||
594 | while (read_count < count) { | |
595 | size_t copy_len, space_left; | |
596 | ||
597 | if (fusionmerge) | |
598 | len = channel_get_next_record(chan, &buf); | |
599 | else | |
600 | len = lib_ring_buffer_get_next_record(chan, buf); | |
601 | len_test: | |
602 | if (len < 0) { | |
603 | /* | |
604 | * Check if buffer is finalized (end of file). | |
605 | */ | |
606 | if (len == -ENODATA) { | |
607 | /* A 0 read_count will tell about end of file */ | |
608 | goto nodata; | |
609 | } | |
610 | if (filp->f_flags & O_NONBLOCK) { | |
611 | if (!read_count) | |
612 | read_count = -EAGAIN; | |
613 | goto nodata; | |
614 | } else { | |
615 | int error; | |
616 | ||
617 | /* | |
618 | * No data available at the moment, return what | |
619 | * we got. | |
620 | */ | |
621 | if (read_count) | |
622 | goto nodata; | |
623 | ||
624 | /* | |
625 | * Wait for returned len to be >= 0 or -ENODATA. | |
626 | */ | |
627 | if (fusionmerge) | |
628 | error = wait_event_interruptible( | |
629 | chan->read_wait, | |
630 | ((len = channel_get_next_record(chan, | |
631 | &buf)), len != -EAGAIN)); | |
632 | else | |
633 | error = wait_event_interruptible( | |
634 | buf->read_wait, | |
635 | ((len = lib_ring_buffer_get_next_record( | |
636 | chan, buf)), len != -EAGAIN)); | |
637 | CHAN_WARN_ON(chan, len == -EBUSY); | |
638 | if (error) { | |
639 | read_count = error; | |
640 | goto nodata; | |
641 | } | |
642 | CHAN_WARN_ON(chan, len < 0 && len != -ENODATA); | |
643 | goto len_test; | |
644 | } | |
645 | } | |
646 | read_offset = buf->iter.read_offset; | |
647 | skip_get_next: | |
648 | space_left = count - read_count; | |
649 | if (len <= space_left) { | |
650 | copy_len = len; | |
651 | chan->iter.len_left = 0; | |
652 | *ppos = 0; | |
653 | } else { | |
654 | copy_len = space_left; | |
655 | chan->iter.len_left = len - copy_len; | |
656 | *ppos = read_offset + copy_len; | |
657 | } | |
658 | if (__lib_ring_buffer_copy_to_user(&buf->backend, read_offset, | |
659 | &user_buf[read_count], | |
660 | copy_len)) { | |
661 | /* | |
662 | * Leave the len_left and ppos values at their current | |
663 | * state, as we currently have a valid event to read. | |
664 | */ | |
665 | return -EFAULT; | |
666 | } | |
667 | read_count += copy_len; | |
668 | }; | |
669 | return read_count; | |
670 | ||
671 | nodata: | |
672 | *ppos = 0; | |
673 | chan->iter.len_left = 0; | |
674 | return read_count; | |
675 | } | |
676 | ||
677 | /** | |
678 | * lib_ring_buffer_file_read - Read buffer record payload. | |
679 | * @filp: file structure pointer. | |
680 | * @buffer: user buffer to read data into. | |
681 | * @count: number of bytes to read. | |
682 | * @ppos: file read position. | |
683 | * | |
684 | * Returns a negative value on error, or the number of bytes read on success. | |
685 | * ppos is used to save the position _within the current record_ between calls | |
686 | * to read(). | |
687 | */ | |
688 | static | |
689 | ssize_t lib_ring_buffer_file_read(struct file *filp, | |
690 | char __user *user_buf, | |
691 | size_t count, | |
692 | loff_t *ppos) | |
693 | { | |
694 | struct inode *inode = filp->f_dentry->d_inode; | |
695 | struct lib_ring_buffer *buf = inode->i_private; | |
696 | struct channel *chan = buf->backend.chan; | |
697 | ||
698 | return channel_ring_buffer_file_read(filp, user_buf, count, ppos, | |
699 | chan, buf, 0); | |
700 | } | |
701 | ||
702 | /** | |
703 | * channel_file_read - Read channel record payload. | |
704 | * @filp: file structure pointer. | |
705 | * @buffer: user buffer to read data into. | |
706 | * @count: number of bytes to read. | |
707 | * @ppos: file read position. | |
708 | * | |
709 | * Returns a negative value on error, or the number of bytes read on success. | |
710 | * ppos is used to save the position _within the current record_ between calls | |
711 | * to read(). | |
712 | */ | |
713 | static | |
714 | ssize_t channel_file_read(struct file *filp, | |
715 | char __user *user_buf, | |
716 | size_t count, | |
717 | loff_t *ppos) | |
718 | { | |
719 | struct inode *inode = filp->f_dentry->d_inode; | |
720 | struct channel *chan = inode->i_private; | |
5a8fd222 | 721 | const struct lib_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
722 | |
723 | if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) | |
724 | return channel_ring_buffer_file_read(filp, user_buf, count, | |
725 | ppos, chan, NULL, 1); | |
726 | else { | |
727 | struct lib_ring_buffer *buf = | |
728 | channel_get_ring_buffer(config, chan, 0); | |
729 | return channel_ring_buffer_file_read(filp, user_buf, count, | |
730 | ppos, chan, buf, 0); | |
731 | } | |
732 | } | |
733 | ||
734 | static | |
735 | int lib_ring_buffer_file_open(struct inode *inode, struct file *file) | |
736 | { | |
737 | struct lib_ring_buffer *buf = inode->i_private; | |
738 | int ret; | |
739 | ||
740 | ret = lib_ring_buffer_iterator_open(buf); | |
741 | if (ret) | |
742 | return ret; | |
743 | ||
744 | file->private_data = buf; | |
745 | ret = nonseekable_open(inode, file); | |
746 | if (ret) | |
747 | goto release_iter; | |
748 | return 0; | |
749 | ||
750 | release_iter: | |
751 | lib_ring_buffer_iterator_release(buf); | |
752 | return ret; | |
753 | } | |
754 | ||
755 | static | |
756 | int lib_ring_buffer_file_release(struct inode *inode, struct file *file) | |
757 | { | |
758 | struct lib_ring_buffer *buf = inode->i_private; | |
759 | ||
760 | lib_ring_buffer_iterator_release(buf); | |
761 | return 0; | |
762 | } | |
763 | ||
764 | static | |
765 | int channel_file_open(struct inode *inode, struct file *file) | |
766 | { | |
767 | struct channel *chan = inode->i_private; | |
768 | int ret; | |
769 | ||
770 | ret = channel_iterator_open(chan); | |
771 | if (ret) | |
772 | return ret; | |
773 | ||
774 | file->private_data = chan; | |
775 | ret = nonseekable_open(inode, file); | |
776 | if (ret) | |
777 | goto release_iter; | |
778 | return 0; | |
779 | ||
780 | release_iter: | |
781 | channel_iterator_release(chan); | |
782 | return ret; | |
783 | } | |
784 | ||
785 | static | |
786 | int channel_file_release(struct inode *inode, struct file *file) | |
787 | { | |
788 | struct channel *chan = inode->i_private; | |
789 | ||
790 | channel_iterator_release(chan); | |
791 | return 0; | |
792 | } | |
793 | ||
794 | const struct file_operations channel_payload_file_operations = { | |
a33c9927 | 795 | .owner = THIS_MODULE, |
f3bc08c5 MD |
796 | .open = channel_file_open, |
797 | .release = channel_file_release, | |
798 | .read = channel_file_read, | |
799 | .llseek = lib_ring_buffer_no_llseek, | |
800 | }; | |
801 | EXPORT_SYMBOL_GPL(channel_payload_file_operations); | |
802 | ||
803 | const struct file_operations lib_ring_buffer_payload_file_operations = { | |
a33c9927 | 804 | .owner = THIS_MODULE, |
f3bc08c5 MD |
805 | .open = lib_ring_buffer_file_open, |
806 | .release = lib_ring_buffer_file_release, | |
807 | .read = lib_ring_buffer_file_read, | |
808 | .llseek = lib_ring_buffer_no_llseek, | |
809 | }; | |
810 | EXPORT_SYMBOL_GPL(lib_ring_buffer_payload_file_operations); |