1 /* SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
6 * Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com>
7 * Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 * Re-using code from kernel/relay.c, which is why it is licensed under
13 #include <linux/module.h>
15 #include <linux/version.h>
17 #include <wrapper/splice.h>
18 #include <wrapper/ringbuffer/backend.h>
19 #include <wrapper/ringbuffer/frontend.h>
20 #include <wrapper/ringbuffer/vfs.h>
23 #define printk_dbg(fmt, args...) printk(fmt, args)
25 #define printk_dbg(fmt, args...)
28 loff_t
vfs_lib_ring_buffer_no_llseek(struct file
*file
, loff_t offset
,
33 EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_no_llseek
);
36 * Release pages from the buffer so splice pipe_to_file can move them.
37 * Called after the pipe has been populated with buffer pages.
39 static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
40 struct pipe_buffer
*pbuf
)
42 __free_page(pbuf
->page
);
45 static const struct pipe_buf_operations ring_buffer_pipe_buf_ops
= {
47 #if (LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0))
48 .map
= generic_pipe_buf_map
,
49 .unmap
= generic_pipe_buf_unmap
,
51 .confirm
= generic_pipe_buf_confirm
,
52 .release
= lib_ring_buffer_pipe_buf_release
,
53 .steal
= generic_pipe_buf_steal
,
54 .get
= generic_pipe_buf_get
,
58 * Page release operation after splice pipe_to_file ends.
60 static void lib_ring_buffer_page_release(struct splice_pipe_desc
*spd
,
63 __free_page(spd
->pages
[i
]);
67 * subbuf_splice_actor - splice up to one subbuf's worth of data
69 static int subbuf_splice_actor(struct file
*in
,
71 struct pipe_inode_info
*pipe
,
74 struct lib_ring_buffer
*buf
)
76 struct channel
*chan
= buf
->backend
.chan
;
77 const struct lib_ring_buffer_config
*config
= &chan
->backend
.config
;
78 unsigned int poff
, subbuf_pages
, nr_pages
;
79 struct page
*pages
[PIPE_DEF_BUFFERS
];
80 struct partial_page partial
[PIPE_DEF_BUFFERS
];
81 struct splice_pipe_desc spd
= {
85 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0))
88 .ops
= &ring_buffer_pipe_buf_ops
,
89 .spd_release
= lib_ring_buffer_page_release
,
91 unsigned long consumed_old
, roffset
;
92 unsigned long bytes_avail
;
95 * Check that a GET_SUBBUF ioctl has been done before.
97 WARN_ON(atomic_long_read(&buf
->active_readers
) != 1);
98 consumed_old
= lib_ring_buffer_get_consumed(config
, buf
);
99 consumed_old
+= *ppos
;
102 * Adjust read len, if longer than what is available.
103 * Max read size is 1 subbuffer due to get_subbuf/put_subbuf for
106 bytes_avail
= chan
->backend
.subbuf_size
;
107 WARN_ON(bytes_avail
> chan
->backend
.buf_size
);
108 len
= min_t(size_t, len
, bytes_avail
);
109 subbuf_pages
= bytes_avail
>> PAGE_SHIFT
;
110 nr_pages
= min_t(unsigned int, subbuf_pages
, PIPE_DEF_BUFFERS
);
111 roffset
= consumed_old
& PAGE_MASK
;
112 poff
= consumed_old
& ~PAGE_MASK
;
113 printk_dbg(KERN_DEBUG
"SPLICE actor len %zu pos %zd write_pos %ld\n",
114 len
, (ssize_t
)*ppos
, lib_ring_buffer_get_offset(config
, buf
));
116 for (; spd
.nr_pages
< nr_pages
; spd
.nr_pages
++) {
117 unsigned int this_len
;
118 unsigned long *pfnp
, new_pfn
;
119 struct page
*new_page
;
124 printk_dbg(KERN_DEBUG
"SPLICE actor loop len %zu roffset %ld\n",
128 * We have to replace the page we are moving into the splice
131 new_page
= alloc_pages_node(cpu_to_node(max(buf
->backend
.cpu
,
133 GFP_KERNEL
| __GFP_ZERO
, 0);
136 new_pfn
= page_to_pfn(new_page
);
137 this_len
= PAGE_SIZE
- poff
;
138 pfnp
= lib_ring_buffer_read_get_pfn(&buf
->backend
, roffset
, &virt
);
139 spd
.pages
[spd
.nr_pages
] = pfn_to_page(*pfnp
);
141 *virt
= page_address(new_page
);
142 spd
.partial
[spd
.nr_pages
].offset
= poff
;
143 spd
.partial
[spd
.nr_pages
].len
= this_len
;
146 roffset
+= PAGE_SIZE
;
153 return wrapper_splice_to_pipe(pipe
, &spd
);
156 ssize_t
lib_ring_buffer_splice_read(struct file
*in
, loff_t
*ppos
,
157 struct pipe_inode_info
*pipe
, size_t len
,
159 struct lib_ring_buffer
*buf
)
161 struct channel
*chan
= buf
->backend
.chan
;
162 const struct lib_ring_buffer_config
*config
= &chan
->backend
.config
;
166 if (config
->output
!= RING_BUFFER_SPLICE
)
170 * We require ppos and length to be page-aligned for performance reasons
171 * (no page copy). Size is known using the ioctl
172 * RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded.
173 * We fail when the ppos or len passed is not page-sized, because splice
174 * is not allowed to copy more than the length passed as parameter (so
175 * the ABI does not let us silently copy more than requested to include
178 if (*ppos
!= PAGE_ALIGN(*ppos
) || len
!= PAGE_ALIGN(len
))
184 printk_dbg(KERN_DEBUG
"SPLICE read len %zu pos %zd\n", len
,
186 while (len
&& !spliced
) {
187 ret
= subbuf_splice_actor(in
, ppos
, pipe
, len
, flags
, buf
);
188 printk_dbg(KERN_DEBUG
"SPLICE read loop ret %d\n", ret
);
192 if (flags
& SPLICE_F_NONBLOCK
)
210 EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read
);
212 ssize_t
vfs_lib_ring_buffer_splice_read(struct file
*in
, loff_t
*ppos
,
213 struct pipe_inode_info
*pipe
, size_t len
,
216 struct lib_ring_buffer
*buf
= in
->private_data
;
218 return lib_ring_buffer_splice_read(in
, ppos
, pipe
, len
, flags
, buf
);
220 EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_splice_read
);