1 /* SPDX-License-Identifier: (GPL-2.0 OR LGPL-2.1)
3 * lib/ringbuffer/backend.h
5 * Ring buffer backend (API).
7 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
10 * the reader in flight recorder mode.
13 #ifndef _LIB_RING_BUFFER_BACKEND_H
14 #define _LIB_RING_BUFFER_BACKEND_H
16 #include <linux/types.h>
17 #include <linux/sched.h>
18 #include <linux/timer.h>
19 #include <linux/wait.h>
20 #include <linux/poll.h>
21 #include <linux/list.h>
24 #include <wrapper/uaccess.h>
25 #include <probes/lttng-probe-user.h>
27 /* Internal helpers */
28 #include <wrapper/ringbuffer/backend_internal.h>
29 #include <wrapper/ringbuffer/frontend_internal.h>
31 /* Ring buffer backend API */
33 /* Ring buffer backend access (read/write) */
35 extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend
*bufb
,
36 size_t offset
, void *dest
, size_t len
);
38 extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend
*bufb
,
39 size_t offset
, void __user
*dest
,
42 extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend
*bufb
,
43 size_t offset
, void *dest
, size_t len
);
45 extern unsigned long *
46 lib_ring_buffer_read_get_pfn(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
50 * Return the address where a given offset is located.
51 * Should be used to get the current subbuffer header pointer. Given we know
52 * it's never on a page boundary, it's safe to write directly to this address,
53 * as long as the write is never bigger than a page size.
56 lib_ring_buffer_offset_address(struct lib_ring_buffer_backend
*bufb
,
59 lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend
*bufb
,
63 * lib_ring_buffer_write - write data to a buffer backend
64 * @config : ring buffer instance configuration
65 * @ctx: ring buffer context. (input arguments only)
66 * @src : source pointer to copy from
67 * @len : length of data to copy
69 * This function copies "len" bytes of data from a source pointer to a buffer
70 * backend, at the current context offset. This is more or less a buffer
71 * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
72 * if copy is crossing a page boundary.
74 static inline __attribute__((always_inline
))
75 void lib_ring_buffer_write(const struct lib_ring_buffer_config
*config
,
76 struct lib_ring_buffer_ctx
*ctx
,
77 const void *src
, size_t len
)
79 struct lib_ring_buffer_backend
*bufb
= &ctx
->buf
->backend
;
80 struct channel_backend
*chanb
= &ctx
->chan
->backend
;
81 size_t index
, pagecpy
;
82 size_t offset
= ctx
->buf_offset
;
83 struct lib_ring_buffer_backend_pages
*backend_pages
;
88 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
89 offset
&= chanb
->buf_size
- 1;
90 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
91 pagecpy
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
92 if (likely(pagecpy
== len
))
93 lib_ring_buffer_do_copy(config
,
94 backend_pages
->p
[index
].virt
95 + (offset
& ~PAGE_MASK
),
98 _lib_ring_buffer_write(bufb
, offset
, src
, len
, 0);
99 ctx
->buf_offset
+= len
;
103 * lib_ring_buffer_memset - write len bytes of c to a buffer backend
104 * @config : ring buffer instance configuration
105 * @bufb : ring buffer backend
106 * @offset : offset within the buffer
107 * @c : the byte to copy
108 * @len : number of bytes to copy
110 * This function writes "len" bytes of "c" to a buffer backend, at a specific
111 * offset. This is more or less a buffer backend-specific memset() operation.
112 * Calls the slow path (_ring_buffer_memset) if write is crossing a page
116 void lib_ring_buffer_memset(const struct lib_ring_buffer_config
*config
,
117 struct lib_ring_buffer_ctx
*ctx
, int c
, size_t len
)
120 struct lib_ring_buffer_backend
*bufb
= &ctx
->buf
->backend
;
121 struct channel_backend
*chanb
= &ctx
->chan
->backend
;
122 size_t index
, pagecpy
;
123 size_t offset
= ctx
->buf_offset
;
124 struct lib_ring_buffer_backend_pages
*backend_pages
;
129 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
130 offset
&= chanb
->buf_size
- 1;
131 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
132 pagecpy
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
133 if (likely(pagecpy
== len
))
134 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
135 + (offset
& ~PAGE_MASK
),
138 _lib_ring_buffer_memset(bufb
, offset
, c
, len
, 0);
139 ctx
->buf_offset
+= len
;
143 * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
144 * terminating character is found in @src. Returns the number of bytes
145 * copied. Does *not* terminate @dest with NULL terminating character.
147 static inline __attribute__((always_inline
))
148 size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config
*config
,
149 char *dest
, const char *src
, size_t len
)
153 for (count
= 0; count
< len
; count
++) {
157 * Only read source character once, in case it is
158 * modified concurrently.
160 c
= LTTNG_READ_ONCE(src
[count
]);
163 lib_ring_buffer_do_copy(config
, &dest
[count
], &c
, 1);
169 * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
170 * terminating character is found in @src, or when a fault occurs.
171 * Returns the number of bytes copied. Does *not* terminate @dest with
172 * NULL terminating character.
174 * This function deals with userspace pointers, it should never be called
175 * directly without having the src pointer checked with access_ok()
178 static inline __attribute__((always_inline
))
179 size_t lib_ring_buffer_do_strcpy_from_user_inatomic(const struct lib_ring_buffer_config
*config
,
180 char *dest
, const char __user
*src
, size_t len
)
184 for (count
= 0; count
< len
; count
++) {
188 ret
= __copy_from_user_inatomic(&c
, src
+ count
, 1);
191 lib_ring_buffer_do_copy(config
, &dest
[count
], &c
, 1);
197 * lib_ring_buffer_strcpy - write string data to a buffer backend
198 * @config : ring buffer instance configuration
199 * @ctx: ring buffer context. (input arguments only)
200 * @src : source pointer to copy from
201 * @len : length of data to copy
202 * @pad : character to use for padding
204 * This function copies @len - 1 bytes of string data from a source
205 * pointer to a buffer backend, followed by a terminating '\0'
206 * character, at the current context offset. This is more or less a
207 * buffer backend-specific strncpy() operation. If a terminating '\0'
208 * character is found in @src before @len - 1 characters are copied, pad
209 * the buffer with @pad characters (e.g. '#'). Calls the slow path
210 * (_ring_buffer_strcpy) if copy is crossing a page boundary.
213 void lib_ring_buffer_strcpy(const struct lib_ring_buffer_config
*config
,
214 struct lib_ring_buffer_ctx
*ctx
,
215 const char *src
, size_t len
, int pad
)
217 struct lib_ring_buffer_backend
*bufb
= &ctx
->buf
->backend
;
218 struct channel_backend
*chanb
= &ctx
->chan
->backend
;
219 size_t index
, pagecpy
;
220 size_t offset
= ctx
->buf_offset
;
221 struct lib_ring_buffer_backend_pages
*backend_pages
;
226 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
227 offset
&= chanb
->buf_size
- 1;
228 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
229 pagecpy
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
230 if (likely(pagecpy
== len
)) {
233 count
= lib_ring_buffer_do_strcpy(config
,
234 backend_pages
->p
[index
].virt
235 + (offset
& ~PAGE_MASK
),
239 if (unlikely(count
< len
- 1)) {
240 size_t pad_len
= len
- 1 - count
;
242 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
243 + (offset
& ~PAGE_MASK
),
248 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
249 + (offset
& ~PAGE_MASK
),
252 _lib_ring_buffer_strcpy(bufb
, offset
, src
, len
, 0, pad
);
254 ctx
->buf_offset
+= len
;
258 * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
259 * @config : ring buffer instance configuration
260 * @ctx: ring buffer context. (input arguments only)
261 * @src : userspace source pointer to copy from
262 * @len : length of data to copy
264 * This function copies "len" bytes of data from a userspace pointer to a
265 * buffer backend, at the current context offset. This is more or less a buffer
266 * backend-specific memcpy() operation. Calls the slow path
267 * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
268 * Disable the page fault handler to ensure we never try to take the mmap_sem.
270 static inline __attribute__((always_inline
))
271 void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config
*config
,
272 struct lib_ring_buffer_ctx
*ctx
,
273 const void __user
*src
, size_t len
)
275 struct lib_ring_buffer_backend
*bufb
= &ctx
->buf
->backend
;
276 struct channel_backend
*chanb
= &ctx
->chan
->backend
;
277 size_t index
, pagecpy
;
278 size_t offset
= ctx
->buf_offset
;
279 struct lib_ring_buffer_backend_pages
*backend_pages
;
285 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
286 offset
&= chanb
->buf_size
- 1;
287 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
288 pagecpy
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
291 if (unlikely(!lttng_access_ok(VERIFY_READ
, src
, len
)))
294 if (likely(pagecpy
== len
)) {
295 ret
= lib_ring_buffer_do_copy_from_user_inatomic(
296 backend_pages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
298 if (unlikely(ret
> 0)) {
303 _lib_ring_buffer_copy_from_user_inatomic(bufb
, offset
, src
, len
, 0);
306 ctx
->buf_offset
+= len
;
313 * In the error path we call the slow path version to avoid
314 * the pollution of static inline code.
316 _lib_ring_buffer_memset(bufb
, offset
, 0, len
, 0);
317 ctx
->buf_offset
+= len
;
321 * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a buffer backend
322 * @config : ring buffer instance configuration
323 * @ctx: ring buffer context (input arguments only)
324 * @src : userspace source pointer to copy from
325 * @len : length of data to copy
326 * @pad : character to use for padding
328 * This function copies @len - 1 bytes of string data from a userspace
329 * source pointer to a buffer backend, followed by a terminating '\0'
330 * character, at the current context offset. This is more or less a
331 * buffer backend-specific strncpy() operation. If a terminating '\0'
332 * character is found in @src before @len - 1 characters are copied, pad
333 * the buffer with @pad characters (e.g. '#'). Calls the slow path
334 * (_ring_buffer_strcpy_from_user_inatomic) if copy is crossing a page
335 * boundary. Disable the page fault handler to ensure we never try to
339 void lib_ring_buffer_strcpy_from_user_inatomic(const struct lib_ring_buffer_config
*config
,
340 struct lib_ring_buffer_ctx
*ctx
,
341 const void __user
*src
, size_t len
, int pad
)
343 struct lib_ring_buffer_backend
*bufb
= &ctx
->buf
->backend
;
344 struct channel_backend
*chanb
= &ctx
->chan
->backend
;
345 size_t index
, pagecpy
;
346 size_t offset
= ctx
->buf_offset
;
347 struct lib_ring_buffer_backend_pages
*backend_pages
;
352 lib_ring_buffer_get_backend_pages_from_ctx(config
, ctx
);
353 offset
&= chanb
->buf_size
- 1;
354 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
355 pagecpy
= min_t(size_t, len
, (-offset
) & ~PAGE_MASK
);
358 if (unlikely(!lttng_access_ok(VERIFY_READ
, src
, len
)))
361 if (likely(pagecpy
== len
)) {
364 count
= lib_ring_buffer_do_strcpy_from_user_inatomic(config
,
365 backend_pages
->p
[index
].virt
366 + (offset
& ~PAGE_MASK
),
370 if (unlikely(count
< len
- 1)) {
371 size_t pad_len
= len
- 1 - count
;
373 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
374 + (offset
& ~PAGE_MASK
),
379 lib_ring_buffer_do_memset(backend_pages
->p
[index
].virt
380 + (offset
& ~PAGE_MASK
),
383 _lib_ring_buffer_strcpy_from_user_inatomic(bufb
, offset
, src
,
387 ctx
->buf_offset
+= len
;
394 * In the error path we call the slow path version to avoid
395 * the pollution of static inline code.
397 _lib_ring_buffer_memset(bufb
, offset
, pad
, len
- 1, 0);
399 _lib_ring_buffer_memset(bufb
, offset
, '\0', 1, 0);
400 ctx
->buf_offset
+= len
;
404 * This accessor counts the number of unread records in a buffer.
405 * It only provides a consistent value if no reads not writes are performed
409 unsigned long lib_ring_buffer_get_records_unread(
410 const struct lib_ring_buffer_config
*config
,
411 struct lib_ring_buffer
*buf
)
413 struct lib_ring_buffer_backend
*bufb
= &buf
->backend
;
414 struct lib_ring_buffer_backend_pages
*pages
;
415 unsigned long records_unread
= 0, sb_bindex
, id
;
418 for (i
= 0; i
< bufb
->chan
->backend
.num_subbuf
; i
++) {
419 id
= bufb
->buf_wsb
[i
].id
;
420 sb_bindex
= subbuffer_id_get_index(config
, id
);
421 pages
= bufb
->array
[sb_bindex
];
422 records_unread
+= v_read(config
, &pages
->records_unread
);
424 if (config
->mode
== RING_BUFFER_OVERWRITE
) {
425 id
= bufb
->buf_rsb
.id
;
426 sb_bindex
= subbuffer_id_get_index(config
, id
);
427 pages
= bufb
->array
[sb_bindex
];
428 records_unread
+= v_read(config
, &pages
->records_unread
);
430 return records_unread
;
434 * We use __copy_from_user_inatomic to copy userspace data after
435 * checking with access_ok() and disabling page faults.
437 * Return 0 if OK, nonzero on error.
440 unsigned long lib_ring_buffer_copy_from_user_check_nofault(void *dest
,
441 const void __user
*src
,
444 return lttng_copy_from_user_check_nofault(dest
, src
, len
);
447 #endif /* _LIB_RING_BUFFER_BACKEND_H */