2 * ring_buffer_backend.c
4 * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; only
9 * version 2.1 of the License.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include <linux/stddef.h>
22 #include <linux/module.h>
23 #include <linux/string.h>
24 #include <linux/bitops.h>
25 #include <linux/delay.h>
26 #include <linux/errno.h>
27 #include <linux/slab.h>
28 #include <linux/cpu.h>
31 #include "../../wrapper/vmalloc.h" /* for wrapper_vmalloc_sync_all() */
32 #include "../../wrapper/ringbuffer/config.h"
33 #include "../../wrapper/ringbuffer/backend.h"
34 #include "../../wrapper/ringbuffer/frontend.h"
37 * lib_ring_buffer_backend_allocate - allocate a channel buffer
38 * @config: ring buffer instance configuration
39 * @buf: the buffer struct
40 * @size: total size of the buffer
41 * @num_subbuf: number of subbuffers
42 * @extra_reader_sb: need extra subbuffer for reader
45 int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config
*config
,
46 struct lib_ring_buffer_backend
*bufb
,
47 size_t size
, size_t num_subbuf
,
50 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
51 unsigned long j
, num_pages
, num_pages_per_subbuf
, page_idx
= 0;
52 unsigned long subbuf_size
, mmap_offset
= 0;
53 unsigned long num_subbuf_alloc
;
58 num_pages
= size
>> PAGE_SHIFT
;
59 num_pages_per_subbuf
= num_pages
>> get_count_order(num_subbuf
);
60 subbuf_size
= chanb
->subbuf_size
;
61 num_subbuf_alloc
= num_subbuf
;
63 if (extra_reader_sb
) {
64 num_pages
+= num_pages_per_subbuf
; /* Add pages for reader */
68 pages
= kmalloc_node(ALIGN(sizeof(*pages
) * num_pages
,
69 1 << INTERNODE_CACHE_SHIFT
),
70 GFP_KERNEL
, cpu_to_node(max(bufb
->cpu
, 0)));
74 virt
= kmalloc_node(ALIGN(sizeof(*virt
) * num_pages
,
75 1 << INTERNODE_CACHE_SHIFT
),
76 GFP_KERNEL
, cpu_to_node(max(bufb
->cpu
, 0)));
80 bufb
->array
= kmalloc_node(ALIGN(sizeof(*bufb
->array
)
82 1 << INTERNODE_CACHE_SHIFT
),
83 GFP_KERNEL
, cpu_to_node(max(bufb
->cpu
, 0)));
84 if (unlikely(!bufb
->array
))
87 for (i
= 0; i
< num_pages
; i
++) {
88 pages
[i
] = alloc_pages_node(cpu_to_node(max(bufb
->cpu
, 0)),
89 GFP_KERNEL
| __GFP_ZERO
, 0);
90 if (unlikely(!pages
[i
]))
92 virt
[i
] = page_address(pages
[i
]);
94 bufb
->num_pages_per_subbuf
= num_pages_per_subbuf
;
96 /* Allocate backend pages array elements */
97 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
100 sizeof(struct lib_ring_buffer_backend_pages
) +
101 sizeof(struct lib_ring_buffer_backend_page
)
102 * num_pages_per_subbuf
,
103 1 << INTERNODE_CACHE_SHIFT
),
104 GFP_KERNEL
, cpu_to_node(max(bufb
->cpu
, 0)));
109 /* Allocate write-side subbuffer table */
110 bufb
->buf_wsb
= kzalloc_node(ALIGN(
111 sizeof(struct lib_ring_buffer_backend_subbuffer
)
113 1 << INTERNODE_CACHE_SHIFT
),
114 GFP_KERNEL
, cpu_to_node(max(bufb
->cpu
, 0)));
115 if (unlikely(!bufb
->buf_wsb
))
118 for (i
= 0; i
< num_subbuf
; i
++)
119 bufb
->buf_wsb
[i
].id
= subbuffer_id(config
, 0, 1, i
);
121 /* Assign read-side subbuffer table */
123 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1,
124 num_subbuf_alloc
- 1);
126 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1, 0);
128 /* Assign pages to page index */
129 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
130 for (j
= 0; j
< num_pages_per_subbuf
; j
++) {
131 CHAN_WARN_ON(chanb
, page_idx
> num_pages
);
132 bufb
->array
[i
]->p
[j
].virt
= virt
[page_idx
];
133 bufb
->array
[i
]->p
[j
].page
= pages
[page_idx
];
136 if (config
->output
== RING_BUFFER_MMAP
) {
137 bufb
->array
[i
]->mmap_offset
= mmap_offset
;
138 mmap_offset
+= subbuf_size
;
143 * If kmalloc ever uses vmalloc underneath, make sure the buffer pages
146 wrapper_vmalloc_sync_all();
152 for (i
= 0; (i
< num_subbuf_alloc
&& bufb
->array
[i
]); i
++)
153 kfree(bufb
->array
[i
]);
155 /* Free all allocated pages */
156 for (i
= 0; (i
< num_pages
&& pages
[i
]); i
++)
157 __free_page(pages
[i
]);
167 int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend
*bufb
,
168 struct channel_backend
*chanb
, int cpu
)
170 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
172 bufb
->chan
= container_of(chanb
, struct channel
, backend
);
175 return lib_ring_buffer_backend_allocate(config
, bufb
, chanb
->buf_size
,
177 chanb
->extra_reader_sb
);
180 void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend
*bufb
)
182 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
183 unsigned long i
, j
, num_subbuf_alloc
;
185 num_subbuf_alloc
= chanb
->num_subbuf
;
186 if (chanb
->extra_reader_sb
)
189 kfree(bufb
->buf_wsb
);
190 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
191 for (j
= 0; j
< bufb
->num_pages_per_subbuf
; j
++)
192 __free_page(bufb
->array
[i
]->p
[j
].page
);
193 kfree(bufb
->array
[i
]);
199 void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend
*bufb
)
201 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
202 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
203 unsigned long num_subbuf_alloc
;
206 num_subbuf_alloc
= chanb
->num_subbuf
;
207 if (chanb
->extra_reader_sb
)
210 for (i
= 0; i
< chanb
->num_subbuf
; i
++)
211 bufb
->buf_wsb
[i
].id
= subbuffer_id(config
, 0, 1, i
);
212 if (chanb
->extra_reader_sb
)
213 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1,
214 num_subbuf_alloc
- 1);
216 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1, 0);
218 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
219 /* Don't reset mmap_offset */
220 v_set(config
, &bufb
->array
[i
]->records_commit
, 0);
221 v_set(config
, &bufb
->array
[i
]->records_unread
, 0);
222 bufb
->array
[i
]->data_size
= 0;
223 /* Don't reset backend page and virt addresses */
225 /* Don't reset num_pages_per_subbuf, cpu, allocated */
226 v_set(config
, &bufb
->records_read
, 0);
230 * The frontend is responsible for also calling ring_buffer_backend_reset for
231 * each buffer when calling channel_backend_reset.
233 void channel_backend_reset(struct channel_backend
*chanb
)
235 struct channel
*chan
= container_of(chanb
, struct channel
, backend
);
236 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
239 * Don't reset buf_size, subbuf_size, subbuf_size_order,
240 * num_subbuf_order, buf_size_order, extra_reader_sb, num_subbuf,
241 * priv, notifiers, config, cpumask and name.
243 chanb
->start_tsc
= config
->cb
.ring_buffer_clock_read(chan
);
246 #ifdef CONFIG_HOTPLUG_CPU
248 * lib_ring_buffer_cpu_hp_callback - CPU hotplug callback
249 * @nb: notifier block
250 * @action: hotplug action to take
253 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
256 int lib_ring_buffer_cpu_hp_callback(struct notifier_block
*nb
,
257 unsigned long action
,
260 unsigned int cpu
= (unsigned long)hcpu
;
261 struct channel_backend
*chanb
= container_of(nb
, struct channel_backend
,
263 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
264 struct lib_ring_buffer
*buf
;
267 CHAN_WARN_ON(chanb
, config
->alloc
== RING_BUFFER_ALLOC_GLOBAL
);
271 case CPU_UP_PREPARE_FROZEN
:
272 buf
= per_cpu_ptr(chanb
->buf
, cpu
);
273 ret
= lib_ring_buffer_create(buf
, chanb
, cpu
);
276 "ring_buffer_cpu_hp_callback: cpu %d "
277 "buffer creation failed\n", cpu
);
282 case CPU_DEAD_FROZEN
:
283 /* No need to do a buffer switch here, because it will happen
284 * when tracing is stopped, or will be done by switch timer CPU
293 * channel_backend_init - initialize a channel backend
294 * @chanb: channel backend
295 * @name: channel name
296 * @config: client ring buffer configuration
297 * @priv: client private data
298 * @parent: dentry of parent directory, %NULL for root directory
299 * @subbuf_size: size of sub-buffers (> PAGE_SIZE, power of 2)
300 * @num_subbuf: number of sub-buffers (power of 2)
302 * Returns channel pointer if successful, %NULL otherwise.
304 * Creates per-cpu channel buffers using the sizes and attributes
305 * specified. The created channel buffer files will be named
306 * name_0...name_N-1. File permissions will be %S_IRUSR.
308 * Called with CPU hotplug disabled.
310 int channel_backend_init(struct channel_backend
*chanb
,
312 const struct lib_ring_buffer_config
*config
,
313 void *priv
, size_t subbuf_size
, size_t num_subbuf
)
315 struct channel
*chan
= container_of(chanb
, struct channel
, backend
);
322 /* Check that the subbuffer size is larger than a page. */
323 if (subbuf_size
< PAGE_SIZE
)
327 * Make sure the number of subbuffers and subbuffer size are
328 * power of 2 and nonzero.
330 if (!subbuf_size
|| (subbuf_size
& (subbuf_size
- 1)))
332 if (!num_subbuf
|| (num_subbuf
& (num_subbuf
- 1)))
335 * Overwrite mode buffers require at least 2 subbuffers per
338 if (config
->mode
== RING_BUFFER_OVERWRITE
&& num_subbuf
< 2)
341 ret
= subbuffer_id_check_index(config
, num_subbuf
);
346 chanb
->buf_size
= num_subbuf
* subbuf_size
;
347 chanb
->subbuf_size
= subbuf_size
;
348 chanb
->buf_size_order
= get_count_order(chanb
->buf_size
);
349 chanb
->subbuf_size_order
= get_count_order(subbuf_size
);
350 chanb
->num_subbuf_order
= get_count_order(num_subbuf
);
351 chanb
->extra_reader_sb
=
352 (config
->mode
== RING_BUFFER_OVERWRITE
) ? 1 : 0;
353 chanb
->num_subbuf
= num_subbuf
;
354 strlcpy(chanb
->name
, name
, NAME_MAX
);
355 memcpy(&chanb
->config
, config
, sizeof(chanb
->config
));
357 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
358 if (!zalloc_cpumask_var(&chanb
->cpumask
, GFP_KERNEL
))
362 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
363 /* Allocating the buffer per-cpu structures */
364 chanb
->buf
= alloc_percpu(struct lib_ring_buffer
);
369 * In case of non-hotplug cpu, if the ring-buffer is allocated
370 * in early initcall, it will not be notified of secondary cpus.
371 * In that off case, we need to allocate for all possible cpus.
373 #ifdef CONFIG_HOTPLUG_CPU
375 * buf->backend.allocated test takes care of concurrent CPU
377 * Priority higher than frontend, so we create the ring buffer
378 * before we start the timer.
380 chanb
->cpu_hp_notifier
.notifier_call
=
381 lib_ring_buffer_cpu_hp_callback
;
382 chanb
->cpu_hp_notifier
.priority
= 5;
383 register_hotcpu_notifier(&chanb
->cpu_hp_notifier
);
386 for_each_online_cpu(i
) {
387 ret
= lib_ring_buffer_create(per_cpu_ptr(chanb
->buf
, i
),
390 goto free_bufs
; /* cpu hotplug locked */
394 for_each_possible_cpu(i
) {
395 ret
= lib_ring_buffer_create(per_cpu_ptr(chanb
->buf
, i
),
398 goto free_bufs
; /* cpu hotplug locked */
402 chanb
->buf
= kzalloc(sizeof(struct lib_ring_buffer
), GFP_KERNEL
);
405 ret
= lib_ring_buffer_create(chanb
->buf
, chanb
, -1);
409 chanb
->start_tsc
= config
->cb
.ring_buffer_clock_read(chan
);
414 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
415 for_each_possible_cpu(i
) {
416 struct lib_ring_buffer
*buf
= per_cpu_ptr(chanb
->buf
, i
);
418 if (!buf
->backend
.allocated
)
420 lib_ring_buffer_free(buf
);
422 #ifdef CONFIG_HOTPLUG_CPU
425 free_percpu(chanb
->buf
);
429 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
)
430 free_cpumask_var(chanb
->cpumask
);
435 * channel_backend_unregister_notifiers - unregister notifiers
440 void channel_backend_unregister_notifiers(struct channel_backend
*chanb
)
442 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
444 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
)
445 unregister_hotcpu_notifier(&chanb
->cpu_hp_notifier
);
449 * channel_backend_free - destroy the channel
452 * Destroy all channel buffers and frees the channel.
454 void channel_backend_free(struct channel_backend
*chanb
)
456 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
459 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
460 for_each_possible_cpu(i
) {
461 struct lib_ring_buffer
*buf
= per_cpu_ptr(chanb
->buf
, i
);
463 if (!buf
->backend
.allocated
)
465 lib_ring_buffer_free(buf
);
467 free_cpumask_var(chanb
->cpumask
);
468 free_percpu(chanb
->buf
);
470 struct lib_ring_buffer
*buf
= chanb
->buf
;
472 CHAN_WARN_ON(chanb
, !buf
->backend
.allocated
);
473 lib_ring_buffer_free(buf
);
479 * lib_ring_buffer_write - write data to a ring_buffer buffer.
480 * @bufb : buffer backend
481 * @offset : offset within the buffer
482 * @src : source address
483 * @len : length to write
484 * @pagecpy : page size copied so far
486 void _lib_ring_buffer_write(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
487 const void *src
, size_t len
, size_t pagecpy
)
489 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
490 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
492 struct lib_ring_buffer_backend_pages
*rpages
;
493 unsigned long sb_bindex
, id
;
499 sbidx
= offset
>> chanb
->subbuf_size_order
;
500 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
503 * Underlying layer should never ask for writes across
506 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
508 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
509 id
= bufb
->buf_wsb
[sbidx
].id
;
510 sb_bindex
= subbuffer_id_get_index(config
, id
);
511 rpages
= bufb
->array
[sb_bindex
];
512 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
513 && subbuffer_id_is_noref(config
, id
));
514 lib_ring_buffer_do_copy(config
,
515 rpages
->p
[index
].virt
516 + (offset
& ~PAGE_MASK
),
518 } while (unlikely(len
!= pagecpy
));
520 EXPORT_SYMBOL_GPL(_lib_ring_buffer_write
);
524 * lib_ring_buffer_memset - write len bytes of c to a ring_buffer buffer.
525 * @bufb : buffer backend
526 * @offset : offset within the buffer
527 * @c : the byte to write
528 * @len : length to write
529 * @pagecpy : page size copied so far
531 void _lib_ring_buffer_memset(struct lib_ring_buffer_backend
*bufb
,
533 int c
, size_t len
, size_t pagecpy
)
535 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
536 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
538 struct lib_ring_buffer_backend_pages
*rpages
;
539 unsigned long sb_bindex
, id
;
544 sbidx
= offset
>> chanb
->subbuf_size_order
;
545 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
548 * Underlying layer should never ask for writes across
551 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
553 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
554 id
= bufb
->buf_wsb
[sbidx
].id
;
555 sb_bindex
= subbuffer_id_get_index(config
, id
);
556 rpages
= bufb
->array
[sb_bindex
];
557 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
558 && subbuffer_id_is_noref(config
, id
));
559 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
560 + (offset
& ~PAGE_MASK
),
562 } while (unlikely(len
!= pagecpy
));
564 EXPORT_SYMBOL_GPL(_lib_ring_buffer_memset
);
567 * lib_ring_buffer_strcpy - write string data to a ring_buffer buffer.
568 * @bufb : buffer backend
569 * @offset : offset within the buffer
570 * @src : source address
571 * @len : length to write
572 * @pagecpy : page size copied so far
573 * @pad : character to use for padding
575 void _lib_ring_buffer_strcpy(struct lib_ring_buffer_backend
*bufb
,
576 size_t offset
, const char *src
, size_t len
,
577 size_t pagecpy
, int pad
)
579 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
580 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
582 struct lib_ring_buffer_backend_pages
*rpages
;
583 unsigned long sb_bindex
, id
;
584 int src_terminated
= 0;
586 CHAN_WARN_ON(chanb
, !len
);
592 sbidx
= offset
>> chanb
->subbuf_size_order
;
593 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
596 * Underlying layer should never ask for writes across
599 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
601 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
602 id
= bufb
->buf_wsb
[sbidx
].id
;
603 sb_bindex
= subbuffer_id_get_index(config
, id
);
604 rpages
= bufb
->array
[sb_bindex
];
605 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
606 && subbuffer_id_is_noref(config
, id
));
608 if (likely(!src_terminated
)) {
609 size_t count
, to_copy
;
613 to_copy
--; /* Final '\0' */
614 count
= lib_ring_buffer_do_strcpy(config
,
615 rpages
->p
[index
].virt
616 + (offset
& ~PAGE_MASK
),
620 if (unlikely(count
< to_copy
)) {
621 size_t pad_len
= to_copy
- count
;
623 /* Next pages will have padding */
625 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
626 + (offset
& ~PAGE_MASK
),
635 pad_len
--; /* Final '\0' */
636 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
637 + (offset
& ~PAGE_MASK
),
641 } while (unlikely(len
!= pagecpy
));
643 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
646 EXPORT_SYMBOL_GPL(_lib_ring_buffer_strcpy
);
649 * lib_ring_buffer_copy_from_user_inatomic - write user data to a ring_buffer buffer.
650 * @bufb : buffer backend
651 * @offset : offset within the buffer
652 * @src : source address
653 * @len : length to write
654 * @pagecpy : page size copied so far
656 * This function deals with userspace pointers, it should never be called
657 * directly without having the src pointer checked with access_ok()
660 void _lib_ring_buffer_copy_from_user_inatomic(struct lib_ring_buffer_backend
*bufb
,
662 const void __user
*src
, size_t len
,
665 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
666 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
668 struct lib_ring_buffer_backend_pages
*rpages
;
669 unsigned long sb_bindex
, id
;
676 sbidx
= offset
>> chanb
->subbuf_size_order
;
677 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
680 * Underlying layer should never ask for writes across
683 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
685 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
686 id
= bufb
->buf_wsb
[sbidx
].id
;
687 sb_bindex
= subbuffer_id_get_index(config
, id
);
688 rpages
= bufb
->array
[sb_bindex
];
689 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
690 && subbuffer_id_is_noref(config
, id
));
691 ret
= lib_ring_buffer_do_copy_from_user_inatomic(rpages
->p
[index
].virt
692 + (offset
& ~PAGE_MASK
),
695 offset
+= (pagecpy
- ret
);
696 len
-= (pagecpy
- ret
);
697 _lib_ring_buffer_memset(bufb
, offset
, 0, len
, 0);
698 break; /* stop copy */
700 } while (unlikely(len
!= pagecpy
));
702 EXPORT_SYMBOL_GPL(_lib_ring_buffer_copy_from_user_inatomic
);
705 * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a ring_buffer buffer.
706 * @bufb : buffer backend
707 * @offset : offset within the buffer
708 * @src : source address
709 * @len : length to write
710 * @pagecpy : page size copied so far
711 * @pad : character to use for padding
713 * This function deals with userspace pointers, it should never be called
714 * directly without having the src pointer checked with access_ok()
717 void _lib_ring_buffer_strcpy_from_user_inatomic(struct lib_ring_buffer_backend
*bufb
,
718 size_t offset
, const char __user
*src
, size_t len
,
719 size_t pagecpy
, int pad
)
721 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
722 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
724 struct lib_ring_buffer_backend_pages
*rpages
;
725 unsigned long sb_bindex
, id
;
726 int src_terminated
= 0;
733 sbidx
= offset
>> chanb
->subbuf_size_order
;
734 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
737 * Underlying layer should never ask for writes across
740 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
742 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
743 id
= bufb
->buf_wsb
[sbidx
].id
;
744 sb_bindex
= subbuffer_id_get_index(config
, id
);
745 rpages
= bufb
->array
[sb_bindex
];
746 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
747 && subbuffer_id_is_noref(config
, id
));
749 if (likely(!src_terminated
)) {
750 size_t count
, to_copy
;
754 to_copy
--; /* Final '\0' */
755 count
= lib_ring_buffer_do_strcpy_from_user_inatomic(config
,
756 rpages
->p
[index
].virt
757 + (offset
& ~PAGE_MASK
),
761 if (unlikely(count
< to_copy
)) {
762 size_t pad_len
= to_copy
- count
;
764 /* Next pages will have padding */
766 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
767 + (offset
& ~PAGE_MASK
),
776 pad_len
--; /* Final '\0' */
777 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
778 + (offset
& ~PAGE_MASK
),
782 } while (unlikely(len
!= pagecpy
));
784 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
787 EXPORT_SYMBOL_GPL(_lib_ring_buffer_strcpy_from_user_inatomic
);
790 * lib_ring_buffer_read - read data from ring_buffer_buffer.
791 * @bufb : buffer backend
792 * @offset : offset within the buffer
793 * @dest : destination address
794 * @len : length to copy to destination
796 * Should be protected by get_subbuf/put_subbuf.
797 * Returns the length copied.
799 size_t lib_ring_buffer_read(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
800 void *dest
, size_t len
)
802 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
803 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
804 size_t index
, pagecpy
, orig_len
;
805 struct lib_ring_buffer_backend_pages
*rpages
;
806 unsigned long sb_bindex
, id
;
809 offset
&= chanb
->buf_size
- 1;
810 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
814 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
815 id
= bufb
->buf_rsb
.id
;
816 sb_bindex
= subbuffer_id_get_index(config
, id
);
817 rpages
= bufb
->array
[sb_bindex
];
818 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
819 && subbuffer_id_is_noref(config
, id
));
820 memcpy(dest
, rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
827 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
829 * Underlying layer should never ask for reads across
832 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
836 EXPORT_SYMBOL_GPL(lib_ring_buffer_read
);
839 * __lib_ring_buffer_copy_to_user - read data from ring_buffer to userspace
840 * @bufb : buffer backend
841 * @offset : offset within the buffer
842 * @dest : destination userspace address
843 * @len : length to copy to destination
845 * Should be protected by get_subbuf/put_subbuf.
846 * access_ok() must have been performed on dest addresses prior to call this
848 * Returns -EFAULT on error, 0 if ok.
850 int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend
*bufb
,
851 size_t offset
, void __user
*dest
, size_t len
)
853 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
854 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
857 struct lib_ring_buffer_backend_pages
*rpages
;
858 unsigned long sb_bindex
, id
;
860 offset
&= chanb
->buf_size
- 1;
861 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
865 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
866 id
= bufb
->buf_rsb
.id
;
867 sb_bindex
= subbuffer_id_get_index(config
, id
);
868 rpages
= bufb
->array
[sb_bindex
];
869 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
870 && subbuffer_id_is_noref(config
, id
));
871 if (__copy_to_user(dest
,
872 rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
880 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
882 * Underlying layer should never ask for reads across
885 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
889 EXPORT_SYMBOL_GPL(__lib_ring_buffer_copy_to_user
);
892 * lib_ring_buffer_read_cstr - read a C-style string from ring_buffer.
893 * @bufb : buffer backend
894 * @offset : offset within the buffer
895 * @dest : destination address
896 * @len : destination's length
898 * Return string's length, or -EINVAL on error.
899 * Should be protected by get_subbuf/put_subbuf.
900 * Destination length should be at least 1 to hold '\0'.
902 int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
903 void *dest
, size_t len
)
905 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
906 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
908 ssize_t pagecpy
, pagelen
, strpagelen
, orig_offset
;
910 struct lib_ring_buffer_backend_pages
*rpages
;
911 unsigned long sb_bindex
, id
;
913 offset
&= chanb
->buf_size
- 1;
914 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
915 orig_offset
= offset
;
919 id
= bufb
->buf_rsb
.id
;
920 sb_bindex
= subbuffer_id_get_index(config
, id
);
921 rpages
= bufb
->array
[sb_bindex
];
922 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
923 && subbuffer_id_is_noref(config
, id
));
924 str
= (char *)rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
925 pagelen
= PAGE_SIZE
- (offset
& ~PAGE_MASK
);
926 strpagelen
= strnlen(str
, pagelen
);
928 pagecpy
= min_t(size_t, len
, strpagelen
);
930 memcpy(dest
, str
, pagecpy
);
935 offset
+= strpagelen
;
936 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
937 if (strpagelen
< pagelen
)
940 * Underlying layer should never ask for reads across
943 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
946 ((char *)dest
)[0] = 0;
947 return offset
- orig_offset
;
949 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_cstr
);
952 * lib_ring_buffer_read_get_page - Get a whole page to read from
953 * @bufb : buffer backend
954 * @offset : offset within the buffer
955 * @virt : pointer to page address (output)
957 * Should be protected by get_subbuf/put_subbuf.
958 * Returns the pointer to the page struct pointer.
960 struct page
**lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend
*bufb
,
961 size_t offset
, void ***virt
)
964 struct lib_ring_buffer_backend_pages
*rpages
;
965 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
966 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
967 unsigned long sb_bindex
, id
;
969 offset
&= chanb
->buf_size
- 1;
970 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
971 id
= bufb
->buf_rsb
.id
;
972 sb_bindex
= subbuffer_id_get_index(config
, id
);
973 rpages
= bufb
->array
[sb_bindex
];
974 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
975 && subbuffer_id_is_noref(config
, id
));
976 *virt
= &rpages
->p
[index
].virt
;
977 return &rpages
->p
[index
].page
;
979 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_get_page
);
982 * lib_ring_buffer_read_offset_address - get address of a buffer location
983 * @bufb : buffer backend
984 * @offset : offset within the buffer.
986 * Return the address where a given offset is located (for read).
987 * Should be used to get the current subbuffer header pointer. Given we know
988 * it's never on a page boundary, it's safe to read/write directly
989 * from/to this address, as long as the read/write is never bigger than a
992 void *lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend
*bufb
,
996 struct lib_ring_buffer_backend_pages
*rpages
;
997 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
998 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
999 unsigned long sb_bindex
, id
;
1001 offset
&= chanb
->buf_size
- 1;
1002 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
1003 id
= bufb
->buf_rsb
.id
;
1004 sb_bindex
= subbuffer_id_get_index(config
, id
);
1005 rpages
= bufb
->array
[sb_bindex
];
1006 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
1007 && subbuffer_id_is_noref(config
, id
));
1008 return rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
1010 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_offset_address
);
1013 * lib_ring_buffer_offset_address - get address of a location within the buffer
1014 * @bufb : buffer backend
1015 * @offset : offset within the buffer.
1017 * Return the address where a given offset is located.
1018 * Should be used to get the current subbuffer header pointer. Given we know
1019 * it's always at the beginning of a page, it's safe to write directly to this
1020 * address, as long as the write is never bigger than a page size.
1022 void *lib_ring_buffer_offset_address(struct lib_ring_buffer_backend
*bufb
,
1025 size_t sbidx
, index
;
1026 struct lib_ring_buffer_backend_pages
*rpages
;
1027 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
1028 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
1029 unsigned long sb_bindex
, id
;
1031 offset
&= chanb
->buf_size
- 1;
1032 sbidx
= offset
>> chanb
->subbuf_size_order
;
1033 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
1034 id
= bufb
->buf_wsb
[sbidx
].id
;
1035 sb_bindex
= subbuffer_id_get_index(config
, id
);
1036 rpages
= bufb
->array
[sb_bindex
];
1037 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
1038 && subbuffer_id_is_noref(config
, id
));
1039 return rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
1041 EXPORT_SYMBOL_GPL(lib_ring_buffer_offset_address
);