[lttng-modules.git] / lib / ringbuffer / backend.h

#ifndef _LINUX_RING_BUFFER_BACKEND_H
#define _LINUX_RING_BUFFER_BACKEND_H

/*
 * linux/ringbuffer/backend.h
 *
 * Copyright (C) 2008-2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Ring buffer backend (API).
 *
 * Dual LGPL v2.1/GPL v2 license.
 *
 * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
 * the reader in flight recorder mode.
 */

#include <linux/types.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/wait.h>
#include <linux/poll.h>
#include <linux/list.h>
#include <linux/fs.h>
#include <linux/mm.h>

/* Internal helpers */
#include "../../wrapper/ringbuffer/backend_internal.h"
#include "../../wrapper/ringbuffer/frontend_internal.h"

/* Ring buffer backend API */

/* Ring buffer backend access (read/write) */

extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
				   size_t offset, void *dest, size_t len);

extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
					  size_t offset, void __user *dest,
					  size_t len);

extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
				     size_t offset, void *dest, size_t len);

extern struct page **
lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
			      void ***virt);

/*
 * Return the address where a given offset is located.
 * Should be used to get the current subbuffer header pointer. Given we know
 * it's never on a page boundary, it's safe to write directly to this address,
 * as long as the write is never bigger than a page size.
 */
extern void *
lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
			       size_t offset);
extern void *
lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
				    size_t offset);

/**
 * lib_ring_buffer_write - write data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a source pointer to a buffer
 * backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
 * if copy is crossing a page boundary.
 */
static inline
void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
			   struct lib_ring_buffer_ctx *ctx,
			   const void *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_copy(config,
					rpages->p[index].virt
					    + (offset & ~PAGE_MASK),
					src, len);
	else
		_lib_ring_buffer_write(bufb, offset, src, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 * @config : ring buffer instance configuration
 * @bufb : ring buffer backend
 * @offset : offset within the buffer
 * @c : the byte to copy
 * @len : number of bytes to copy
 *
 * This function writes "len" bytes of "c" to a buffer backend, at a specific
 * offset. This is more or less a buffer backend-specific memset() operation.
 * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 * boundary.
 */
static inline
void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
			    struct lib_ring_buffer_ctx *ctx, int c, size_t len)
{

	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_memset(rpages->p[index].virt
					  + (offset & ~PAGE_MASK),
					  c, len);
	else
		_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_copy_from_user - write userspace data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : userspace source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a userspace pointer to a
 * buffer backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path
 * (_ring_buffer_write_from_user) if copy is crossing a page boundary.
 */
static inline
void lib_ring_buffer_copy_from_user(const struct lib_ring_buffer_config *config,
				    struct lib_ring_buffer_ctx *ctx,
				    const void __user *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;
	unsigned long ret;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));

	if (unlikely(!access_ok(VERIFY_READ, src, len)))
		goto fill_buffer;

	if (likely(pagecpy == len)) {
		ret = lib_ring_buffer_do_copy_from_user(
			rpages->p[index].virt + (offset & ~PAGE_MASK),
			src, len);
		if (unlikely(ret > 0)) {
			len -= (pagecpy - ret);
			offset += (pagecpy - ret);
			goto fill_buffer;
		}
	} else {
		_lib_ring_buffer_copy_from_user(bufb, offset, src, len, 0);
	}
	ctx->buf_offset += len;

	return;

fill_buffer:
	/*
	 * In the error path we call the slow path version to avoid
	 * the pollution of static inline code.
	 */
	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
}

/*
 * This accessor counts the number of unread records in a buffer.
 * It only provides a consistent value if no reads not writes are performed
 * concurrently.
 */
static inline
unsigned long lib_ring_buffer_get_records_unread(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	struct lib_ring_buffer_backend *bufb = &buf->backend;
	struct lib_ring_buffer_backend_pages *pages;
	unsigned long records_unread = 0, sb_bindex, id;
	unsigned int i;

	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
		id = bufb->buf_wsb[i].id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	if (config->mode == RING_BUFFER_OVERWRITE) {
		id = bufb->buf_rsb.id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	return records_unread;
}

ssize_t lib_ring_buffer_file_splice_read(struct file *in, loff_t *ppos,
					 struct pipe_inode_info *pipe,
					 size_t len, unsigned int flags);
loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);

#endif /* _LINUX_RING_BUFFER_BACKEND_H */
Commit	Line	Data
f3bc08c5 MD	1	#ifndef _LINUX_RING_BUFFER_BACKEND_H
	2	#define _LINUX_RING_BUFFER_BACKEND_H
	3
	4	/*
	5	* linux/ringbuffer/backend.h
	6	*
	7	* Copyright (C) 2008-2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	8	*
	9	* Ring buffer backend (API).
	10	*
	11	* Dual LGPL v2.1/GPL v2 license.
	12	*
	13	* Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
	14	* the reader in flight recorder mode.
	15	*/
	16
	17	#include <linux/types.h>
	18	#include <linux/sched.h>
	19	#include <linux/timer.h>
	20	#include <linux/wait.h>
	21	#include <linux/poll.h>
	22	#include <linux/list.h>
	23	#include <linux/fs.h>
	24	#include <linux/mm.h>
	25
	26	/* Internal helpers */
	27	#include "../../wrapper/ringbuffer/backend_internal.h"
	28	#include "../../wrapper/ringbuffer/frontend_internal.h"
	29
	30	/* Ring buffer backend API */
	31
	32	/* Ring buffer backend access (read/write) */
	33
	34	extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
	35	size_t offset, void *dest, size_t len);
	36
	37	extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
	38	size_t offset, void __user *dest,
	39	size_t len);
	40
	41	extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
	42	size_t offset, void *dest, size_t len);
	43
	44	extern struct page **
	45	lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
	46	void ***virt);
	47
	48	/*
	49	* Return the address where a given offset is located.
	50	* Should be used to get the current subbuffer header pointer. Given we know
	51	* it's never on a page boundary, it's safe to write directly to this address,
	52	* as long as the write is never bigger than a page size.
	53	*/
	54	extern void *
	55	lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
	56	size_t offset);
	57	extern void *
	58	lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
	59	size_t offset);
	60
	61	/**
	62	* lib_ring_buffer_write - write data to a buffer backend
	63	* @config : ring buffer instance configuration
	64	* @ctx: ring buffer context. (input arguments only)
65	* @src : source pointer to copy from
66	* @len : length of data to copy
67	*
68	* This function copies "len" bytes of data from a source pointer to a buffer
69	* backend, at the current context offset. This is more or less a buffer
70	* backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
71	* if copy is crossing a page boundary.
72	*/
73	static inline
74	void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
75	struct lib_ring_buffer_ctx *ctx,
76	const void *src, size_t len)
77	{
78	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
79	struct channel_backend *chanb = &ctx->chan->backend;
80	size_t sbidx, index;
81	size_t offset = ctx->buf_offset;
82	ssize_t pagecpy;
83	struct lib_ring_buffer_backend_pages *rpages;
84	unsigned long sb_bindex, id;
85
86	offset &= chanb->buf_size - 1;
87	sbidx = offset >> chanb->subbuf_size_order;
88	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
89	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
90	id = bufb->buf_wsb[sbidx].id;
91	sb_bindex = subbuffer_id_get_index(config, id);
92	rpages = bufb->array[sb_bindex];
93	CHAN_WARN_ON(ctx->chan,
94	config->mode == RING_BUFFER_OVERWRITE
95	&& subbuffer_id_is_noref(config, id));
96	if (likely(pagecpy == len))
97	lib_ring_buffer_do_copy(config,
98	rpages->p[index].virt
99	+ (offset & ~PAGE_MASK),
100	src, len);
101	else
102	_lib_ring_buffer_write(bufb, offset, src, len, 0);
103	ctx->buf_offset += len;
104	}
105
4ea00e4f JD	106	/**
	107	* lib_ring_buffer_memset - write len bytes of c to a buffer backend
	108	* @config : ring buffer instance configuration
	109	* @bufb : ring buffer backend
	110	* @offset : offset within the buffer
	111	* @c : the byte to copy
	112	* @len : number of bytes to copy
	113	*
	114	* This function writes "len" bytes of "c" to a buffer backend, at a specific
	115	* offset. This is more or less a buffer backend-specific memset() operation.
	116	* Calls the slow path (_ring_buffer_memset) if write is crossing a page
	117	* boundary.
	118	*/
	119	static inline
	120	void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
	121	struct lib_ring_buffer_ctx *ctx, int c, size_t len)
	122	{
	123
	124	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	125	struct channel_backend *chanb = &ctx->chan->backend;
	126	size_t sbidx, index;
	127	size_t offset = ctx->buf_offset;
	128	ssize_t pagecpy;
	129	struct lib_ring_buffer_backend_pages *rpages;
	130	unsigned long sb_bindex, id;
	131
	132	offset &= chanb->buf_size - 1;
	133	sbidx = offset >> chanb->subbuf_size_order;
	134	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	135	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	136	id = bufb->buf_wsb[sbidx].id;
	137	sb_bindex = subbuffer_id_get_index(config, id);
	138	rpages = bufb->array[sb_bindex];
	139	CHAN_WARN_ON(ctx->chan,
	140	config->mode == RING_BUFFER_OVERWRITE
	141	&& subbuffer_id_is_noref(config, id));
	142	if (likely(pagecpy == len))
	143	lib_ring_buffer_do_memset(rpages->p[index].virt
	144	+ (offset & ~PAGE_MASK),
	145	c, len);
	146	else
	147	_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	148	ctx->buf_offset += len;
	149	}
	150
	151	/**
	152	* lib_ring_buffer_copy_from_user - write userspace data to a buffer backend
	153	* @config : ring buffer instance configuration
	154	* @ctx: ring buffer context. (input arguments only)
	155	* @src : userspace source pointer to copy from
	156	* @len : length of data to copy
	157	*
	158	* This function copies "len" bytes of data from a userspace pointer to a
	159	* buffer backend, at the current context offset. This is more or less a buffer
	160	* backend-specific memcpy() operation. Calls the slow path
	161	* (_ring_buffer_write_from_user) if copy is crossing a page boundary.
	162	*/
	163	static inline
	164	void lib_ring_buffer_copy_from_user(const struct lib_ring_buffer_config *config,
	165	struct lib_ring_buffer_ctx *ctx,
	166	const void __user *src, size_t len)
	167	{
	168	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	169	struct channel_backend *chanb = &ctx->chan->backend;
170	size_t sbidx, index;
171	size_t offset = ctx->buf_offset;
172	ssize_t pagecpy;
173	struct lib_ring_buffer_backend_pages *rpages;
174	unsigned long sb_bindex, id;
175	unsigned long ret;
176
177	offset &= chanb->buf_size - 1;
178	sbidx = offset >> chanb->subbuf_size_order;
179	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
180	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
181	id = bufb->buf_wsb[sbidx].id;
182	sb_bindex = subbuffer_id_get_index(config, id);
183	rpages = bufb->array[sb_bindex];
184	CHAN_WARN_ON(ctx->chan,
185	config->mode == RING_BUFFER_OVERWRITE
186	&& subbuffer_id_is_noref(config, id));
187
188	if (unlikely(!access_ok(VERIFY_READ, src, len)))
189	goto fill_buffer;
190
191	if (likely(pagecpy == len)) {
192	ret = lib_ring_buffer_do_copy_from_user(
193	rpages->p[index].virt + (offset & ~PAGE_MASK),
194	src, len);
195	if (unlikely(ret > 0)) {
196	len -= (pagecpy - ret);
197	offset += (pagecpy - ret);
198	goto fill_buffer;
199	}
200	} else {
201	_lib_ring_buffer_copy_from_user(bufb, offset, src, len, 0);
202	}
203	ctx->buf_offset += len;
204
205	return;
206
207	fill_buffer:
208	/*
209	* In the error path we call the slow path version to avoid
210	* the pollution of static inline code.
211	*/
212	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
213	}
214
f3bc08c5 MD	215	/*
	216	* This accessor counts the number of unread records in a buffer.
	217	* It only provides a consistent value if no reads not writes are performed
	218	* concurrently.
	219	*/
	220	static inline
	221	unsigned long lib_ring_buffer_get_records_unread(
	222	const struct lib_ring_buffer_config *config,
	223	struct lib_ring_buffer *buf)
	224	{
	225	struct lib_ring_buffer_backend *bufb = &buf->backend;
	226	struct lib_ring_buffer_backend_pages *pages;
	227	unsigned long records_unread = 0, sb_bindex, id;
	228	unsigned int i;
	229
	230	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
	231	id = bufb->buf_wsb[i].id;
	232	sb_bindex = subbuffer_id_get_index(config, id);
	233	pages = bufb->array[sb_bindex];
	234	records_unread += v_read(config, &pages->records_unread);
	235	}
	236	if (config->mode == RING_BUFFER_OVERWRITE) {
	237	id = bufb->buf_rsb.id;
	238	sb_bindex = subbuffer_id_get_index(config, id);
	239	pages = bufb->array[sb_bindex];
	240	records_unread += v_read(config, &pages->records_unread);
	241	}
	242	return records_unread;
	243	}
	244
	245	ssize_t lib_ring_buffer_file_splice_read(struct file in, loff_t ppos,
	246	struct pipe_inode_info *pipe,
	247	size_t len, unsigned int flags);
	248	loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);
	249
	250	#endif /* _LINUX_RING_BUFFER_BACKEND_H */