1 /* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
5 * Ring buffer backend (API).
7 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
10 * the reader in flight recorder mode.
13 #ifndef _LIB_RING_BUFFER_BACKEND_H
14 #define _LIB_RING_BUFFER_BACKEND_H
16 #include <linux/types.h>
17 #include <linux/sched.h>
18 #include <linux/timer.h>
19 #include <linux/wait.h>
20 #include <linux/poll.h>
21 #include <linux/list.h>
24 #include <wrapper/uaccess.h>
25 #include <lttng/probe-user.h>
27 /* Internal helpers */
28 #include <ringbuffer/backend_internal.h>
29 #include <ringbuffer/frontend_internal.h>
31 /* Ring buffer backend API */
33 /* Ring buffer backend access (read/write) */
35 extern size_t lib_ring_buffer_read(struct lttng_kernel_ring_buffer_backend
*bufb
,
36 size_t offset
, void *dest
, size_t len
);
38 extern int __lib_ring_buffer_copy_to_user(struct lttng_kernel_ring_buffer_backend
*bufb
,
39 size_t offset
, void __user
*dest
,
42 extern int lib_ring_buffer_read_cstr(struct lttng_kernel_ring_buffer_backend
*bufb
,
43 size_t offset
, void *dest
, size_t len
);
45 extern unsigned long *
46 lib_ring_buffer_read_get_pfn(struct lttng_kernel_ring_buffer_backend
*bufb
, size_t offset
,
50 * Return the address where a given offset is located.
51 * Should be used to get the current subbuffer header pointer. Given we know
52 * it's never on a page boundary, it's safe to write directly to this address,
53 * as long as the write is never bigger than a page size.
56 lib_ring_buffer_offset_address(struct lttng_kernel_ring_buffer_backend
*bufb
,
59 lib_ring_buffer_read_offset_address(struct lttng_kernel_ring_buffer_backend
*bufb
,
63 * lib_ring_buffer_write - write data to a buffer backend
64 * @config : ring buffer instance configuration
65 * @ctx: ring buffer context. (input arguments only)
66 * @src : source pointer to copy from
67 * @len : length of data to copy
69 * This function copies "len" bytes of data from a source pointer to a buffer
70 * backend, at the current context offset. This is more or less a buffer
71 * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
72 * if copy is crossing a page boundary.
74 static inline __attribute__((always_inline
))
75 void lib_ring_buffer_write(const struct lttng_kernel_ring_buffer_config
*config
,
76 struct lttng_kernel_ring_buffer_ctx
*ctx
,
77 const void *src
, size_t len
)
79 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
80 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
81 size_t index
, bytes_left_in_page
;
82 size_t offset
= ctx
->priv
.buf_offset
;
83 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
88 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
89 offset
&= chanb
->buf_size
- 1;
90 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
91 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
92 if (likely(bytes_left_in_page
== len
))
93 lib_ring_buffer_do_copy(config
,
94 backend_pages
->p
[index
].virt
95 + (offset
& ~PAGE_MASK
),
98 _lib_ring_buffer_write(bufb
, offset
, src
, len
);
99 ctx
->priv
.buf_offset
+= len
;
103 * lib_ring_buffer_memset - write len bytes of c to a buffer backend
104 * @config : ring buffer instance configuration
105 * @bufb : ring buffer backend
106 * @offset : offset within the buffer
107 * @c : the byte to copy
108 * @len : number of bytes to copy
110 * This function writes "len" bytes of "c" to a buffer backend, at a specific
111 * offset. This is more or less a buffer backend-specific memset() operation.
112 * Calls the slow path (_ring_buffer_memset) if write is crossing a page
116 void lib_ring_buffer_memset(const struct lttng_kernel_ring_buffer_config
*config
,
117 struct lttng_kernel_ring_buffer_ctx
*ctx
, int c
, size_t len
)
120 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
121 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
122 size_t index
, bytes_left_in_page
;
123 size_t offset
= ctx
->priv
.buf_offset
;
124 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
129 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
130 offset
&= chanb
->buf_size
- 1;
131 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
132 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
133 if (likely(bytes_left_in_page
== len
))
134 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
135 + (offset
& ~PAGE_MASK
),
138 _lib_ring_buffer_memset(bufb
, offset
, c
, len
);
139 ctx
->priv
.buf_offset
+= len
;
143 * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
144 * terminating character is found in @src. Returns the number of bytes
145 * copied. Does *not* terminate @dest with NULL terminating character.
147 static inline __attribute__((always_inline
))
148 size_t lib_ring_buffer_do_strcpy(const struct lttng_kernel_ring_buffer_config
*config
,
149 char *dest
, const char *src
, size_t len
)
153 for (count
= 0; count
< len
; count
++) {
157 * Only read source character once, in case it is
158 * modified concurrently.
160 c
= LTTNG_READ_ONCE(src
[count
]);
163 lib_ring_buffer_do_copy(config
, &dest
[count
], &c
, 1);
169 * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
170 * terminating character is found in @src, or when a fault occurs.
171 * Returns the number of bytes copied. Does *not* terminate @dest with
172 * NULL terminating character.
174 * This function deals with userspace pointers, it should never be called
175 * directly without having the src pointer checked with access_ok()
178 static inline __attribute__((always_inline
))
179 size_t lib_ring_buffer_do_strcpy_from_user_inatomic(const struct lttng_kernel_ring_buffer_config
*config
,
180 char *dest
, const char __user
*src
, size_t len
)
184 for (count
= 0; count
< len
; count
++) {
188 ret
= __copy_from_user_inatomic(&c
, src
+ count
, 1);
191 lib_ring_buffer_do_copy(config
, &dest
[count
], &c
, 1);
197 * lib_ring_buffer_strcpy - write string data to a buffer backend
198 * @config : ring buffer instance configuration
199 * @ctx: ring buffer context. (input arguments only)
200 * @src : source pointer to copy from
201 * @len : length of data to copy
202 * @pad : character to use for padding
204 * This function copies @len - 1 bytes of string data from a source
205 * pointer to a buffer backend, followed by a terminating '\0'
206 * character, at the current context offset. This is more or less a
207 * buffer backend-specific strncpy() operation. If a terminating '\0'
208 * character is found in @src before @len - 1 characters are copied, pad
209 * the buffer with @pad characters (e.g. '#'). Calls the slow path
210 * (_ring_buffer_strcpy) if copy is crossing a page boundary.
213 void lib_ring_buffer_strcpy(const struct lttng_kernel_ring_buffer_config
*config
,
214 struct lttng_kernel_ring_buffer_ctx
*ctx
,
215 const char *src
, size_t len
, int pad
)
217 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
218 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
219 size_t index
, bytes_left_in_page
;
220 size_t offset
= ctx
->priv
.buf_offset
;
221 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
226 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
227 offset
&= chanb
->buf_size
- 1;
228 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
229 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
230 if (likely(bytes_left_in_page
== len
)) {
233 count
= lib_ring_buffer_do_strcpy(config
,
234 backend_pages
->p
[index
].virt
235 + (offset
& ~PAGE_MASK
),
239 if (unlikely(count
< len
- 1)) {
240 size_t pad_len
= len
- 1 - count
;
242 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
243 + (offset
& ~PAGE_MASK
),
248 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
249 + (offset
& ~PAGE_MASK
),
252 _lib_ring_buffer_strcpy(bufb
, offset
, src
, len
, pad
);
254 ctx
->priv
.buf_offset
+= len
;
258 * lib_ring_buffer_pstrcpy - write kernel C-string (input) to a buffer backend P-string
259 * @config : ring buffer instance configuration
260 * @ctx: ring buffer context. (input arguments only)
261 * @src : source pointer to copy from
262 * @len : length of data to copy
263 * @pad : character to use for padding
265 * This function copies up to @len bytes of data from a source pointer
266 * to a Pascal String into the buffer backend. If a terminating '\0'
267 * character is found in @src before @len characters are copied, pad the
268 * buffer with @pad characters (e.g. '\0').
270 * The length of the pascal strings in the ring buffer is explicit: it
271 * is either the array or sequence length.
274 void lib_ring_buffer_pstrcpy(const struct lttng_kernel_ring_buffer_config
*config
,
275 struct lttng_kernel_ring_buffer_ctx
*ctx
,
276 const char *src
, size_t len
, char pad
)
277 __attribute__((always_inline
));
279 void lib_ring_buffer_pstrcpy(const struct lttng_kernel_ring_buffer_config
*config
,
280 struct lttng_kernel_ring_buffer_ctx
*ctx
,
281 const char *src
, size_t len
, char pad
)
283 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
284 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
285 size_t index
, bytes_left_in_page
;
286 size_t offset
= ctx
->priv
.buf_offset
;
287 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
292 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
293 offset
&= chanb
->buf_size
- 1;
294 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
295 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
296 if (likely(bytes_left_in_page
== len
)) {
299 count
= lib_ring_buffer_do_strcpy(config
,
300 backend_pages
->p
[index
].virt
301 + (offset
& ~PAGE_MASK
),
305 if (unlikely(count
< len
)) {
306 size_t pad_len
= len
- count
;
308 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
309 + (offset
& ~PAGE_MASK
),
314 _lib_ring_buffer_pstrcpy(bufb
, offset
, src
, len
, pad
);
316 ctx
->priv
.buf_offset
+= len
;
320 * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
321 * @config : ring buffer instance configuration
322 * @ctx: ring buffer context. (input arguments only)
323 * @src : userspace source pointer to copy from
324 * @len : length of data to copy
326 * This function copies "len" bytes of data from a userspace pointer to a
327 * buffer backend, at the current context offset. This is more or less a buffer
328 * backend-specific memcpy() operation. Calls the slow path
329 * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
330 * Disable the page fault handler to ensure we never try to take the mmap_sem.
332 static inline __attribute__((always_inline
))
333 void lib_ring_buffer_copy_from_user_inatomic(const struct lttng_kernel_ring_buffer_config
*config
,
334 struct lttng_kernel_ring_buffer_ctx
*ctx
,
335 const void __user
*src
, size_t len
)
337 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
338 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
339 size_t index
, bytes_left_in_page
;
340 size_t offset
= ctx
->priv
.buf_offset
;
341 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
347 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
348 offset
&= chanb
->buf_size
- 1;
349 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
350 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
352 if (unlikely(!lttng_access_ok(VERIFY_READ
, src
, len
)))
356 if (likely(bytes_left_in_page
== len
)) {
357 ret
= lib_ring_buffer_do_copy_from_user_inatomic(
358 backend_pages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
360 if (unlikely(ret
> 0)) {
362 goto fill_buffer_enable_pf
;
365 _lib_ring_buffer_copy_from_user_inatomic(bufb
, offset
, src
, len
);
368 ctx
->priv
.buf_offset
+= len
;
372 fill_buffer_enable_pf
:
376 * In the error path we call the slow path version to avoid
377 * the pollution of static inline code.
379 _lib_ring_buffer_memset(bufb
, offset
, 0, len
);
380 ctx
->priv
.buf_offset
+= len
;
384 * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a buffer backend
385 * @config : ring buffer instance configuration
386 * @ctx: ring buffer context (input arguments only)
387 * @src : userspace source pointer to copy from
388 * @len : length of data to copy
389 * @pad : character to use for padding
391 * This function copies @len - 1 bytes of string data from a userspace
392 * source pointer to a buffer backend, followed by a terminating '\0'
393 * character, at the current context offset. This is more or less a
394 * buffer backend-specific strncpy() operation. If a terminating '\0'
395 * character is found in @src before @len - 1 characters are copied, pad
396 * the buffer with @pad characters (e.g. '#'). Calls the slow path
397 * (_ring_buffer_strcpy_from_user_inatomic) if copy is crossing a page
398 * boundary. Disable the page fault handler to ensure we never try to
402 void lib_ring_buffer_strcpy_from_user_inatomic(const struct lttng_kernel_ring_buffer_config
*config
,
403 struct lttng_kernel_ring_buffer_ctx
*ctx
,
404 const void __user
*src
, size_t len
, int pad
)
406 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
407 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
408 size_t index
, bytes_left_in_page
;
409 size_t offset
= ctx
->priv
.buf_offset
;
410 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
415 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
416 offset
&= chanb
->buf_size
- 1;
417 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
418 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
420 if (unlikely(!lttng_access_ok(VERIFY_READ
, src
, len
)))
424 if (likely(bytes_left_in_page
== len
)) {
427 count
= lib_ring_buffer_do_strcpy_from_user_inatomic(config
,
428 backend_pages
->p
[index
].virt
429 + (offset
& ~PAGE_MASK
),
433 if (unlikely(count
< len
- 1)) {
434 size_t pad_len
= len
- 1 - count
;
436 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
437 + (offset
& ~PAGE_MASK
),
442 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
443 + (offset
& ~PAGE_MASK
),
446 _lib_ring_buffer_strcpy_from_user_inatomic(bufb
, offset
, src
,
450 ctx
->priv
.buf_offset
+= len
;
456 * In the error path we call the slow path version to avoid
457 * the pollution of static inline code.
459 _lib_ring_buffer_memset(bufb
, offset
, pad
, len
- 1);
461 _lib_ring_buffer_memset(bufb
, offset
, '\0', 1);
462 ctx
->priv
.buf_offset
+= len
;
466 * lib_ring_buffer_pstrcpy_from_user_inatomic - write user-space C-string (input) to a buffer backend P-string
467 * @config : ring buffer instance configuration
468 * @ctx: ring buffer context. (input arguments only)
469 * @src : source pointer to copy from
470 * @len : length of data to copy
471 * @pad : character to use for padding
473 * This function copies up to @len bytes of data from a source pointer
474 * to a Pascal String into the buffer backend. If a terminating '\0'
475 * character is found in @src before @len characters are copied, pad the
476 * buffer with @pad characters (e.g. '\0').
478 * The length of the pascal strings in the ring buffer is explicit: it
479 * is either the array or sequence length.
482 void lib_ring_buffer_pstrcpy_from_user_inatomic(const struct lttng_kernel_ring_buffer_config
*config
,
483 struct lttng_kernel_ring_buffer_ctx
*ctx
,
484 const char __user
*src
, size_t len
, char pad
)
485 __attribute__((always_inline
));
487 void lib_ring_buffer_pstrcpy_from_user_inatomic(const struct lttng_kernel_ring_buffer_config
*config
,
488 struct lttng_kernel_ring_buffer_ctx
*ctx
,
489 const char __user
*src
, size_t len
, char pad
)
491 struct lttng_kernel_ring_buffer_backend
*bufb
= &ctx
->priv
.buf
->backend
;
492 struct channel_backend
*chanb
= &ctx
->priv
.chan
->backend
;
493 size_t index
, bytes_left_in_page
;
494 size_t offset
= ctx
->priv
.buf_offset
;
495 struct lttng_kernel_ring_buffer_backend_pages
*backend_pages
;
500 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
501 offset
&= chanb
->buf_size
- 1;
502 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
503 bytes_left_in_page
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
505 if (unlikely(!lttng_access_ok(VERIFY_READ
, src
, len
)))
509 if (likely(bytes_left_in_page
== len
)) {
512 count
= lib_ring_buffer_do_strcpy_from_user_inatomic(config
,
513 backend_pages
->p
[index
].virt
514 + (offset
& ~PAGE_MASK
),
518 if (unlikely(count
< len
)) {
519 size_t pad_len
= len
- count
;
521 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
522 + (offset
& ~PAGE_MASK
),
527 _lib_ring_buffer_pstrcpy_from_user_inatomic(bufb
, offset
, src
, len
, pad
);
529 ctx
->priv
.buf_offset
+= len
;
536 * In the error path we call the slow path version to avoid
537 * the pollution of static inline code.
539 _lib_ring_buffer_memset(bufb
, offset
, pad
, len
);
540 ctx
->priv
.buf_offset
+= len
;
544 * This accessor counts the number of unread records in a buffer.
545 * It only provides a consistent value if no reads not writes are performed
549 unsigned long lib_ring_buffer_get_records_unread(
550 const struct lttng_kernel_ring_buffer_config
*config
,
551 struct lttng_kernel_ring_buffer
*buf
)
553 struct lttng_kernel_ring_buffer_backend
*bufb
= &buf
->backend
;
554 struct lttng_kernel_ring_buffer_backend_pages
*pages
;
555 unsigned long records_unread
= 0, sb_bindex
, id
;
558 for (i
= 0; i
< bufb
->chan
->backend
.num_subbuf
; i
++) {
559 id
= bufb
->buf_wsb
[i
].id
;
560 sb_bindex
= subbuffer_id_get_index(config
, id
);
561 pages
= bufb
->array
[sb_bindex
];
562 records_unread
+= v_read(config
, &pages
->records_unread
);
564 if (config
->mode
== RING_BUFFER_OVERWRITE
) {
565 id
= bufb
->buf_rsb
.id
;
566 sb_bindex
= subbuffer_id_get_index(config
, id
);
567 pages
= bufb
->array
[sb_bindex
];
568 records_unread
+= v_read(config
, &pages
->records_unread
);
570 return records_unread
;
574 * We use __copy_from_user_inatomic to copy userspace data after
575 * checking with access_ok() and disabling page faults.
577 * Return 0 if OK, nonzero on error.
580 unsigned long lib_ring_buffer_copy_from_user_check_nofault(void *dest
,
581 const void __user
*src
,
584 return lttng_copy_from_user_check_nofault(dest
, src
, len
);
587 #endif /* _LIB_RING_BUFFER_BACKEND_H */