Commit | Line | Data |
---|---|---|
1c8284eb MD |
1 | /* |
2 | * include/linux/ltt-relay.h | |
3 | * | |
4 | * Copyright (C) 2008,2009 - Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> | |
5 | * | |
6 | * Dual LGPL v2.1/GPL v2 license. | |
7 | * | |
8 | * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by | |
9 | * the reader in flight recorder mode. | |
10 | */ | |
11 | ||
12 | #ifndef _LINUX_LTT_RELAY_H | |
13 | #define _LINUX_LTT_RELAY_H | |
14 | ||
15 | #include <linux/types.h> | |
16 | #include <linux/sched.h> | |
17 | #include <linux/timer.h> | |
18 | #include <linux/wait.h> | |
19 | #include <linux/fs.h> | |
20 | #include <linux/poll.h> | |
21 | #include <linux/kref.h> | |
22 | #include <linux/mm.h> | |
23 | #include <linux/ltt-channels.h> | |
24 | ||
25 | #include "ltt-tracer-core.h" | |
26 | ||
27 | /* Use lowest pointer bit to show the sub-buffer has no reference. */ | |
28 | #define RCHAN_NOREF_FLAG 0x1UL | |
29 | ||
30 | #define RCHAN_SB_IS_NOREF(x) ((unsigned long)(x) & RCHAN_NOREF_FLAG) | |
31 | #define RCHAN_SB_SET_NOREF(x) \ | |
32 | (x = (struct chanbuf_page *)((unsigned long)(x) | RCHAN_NOREF_FLAG)) | |
33 | #define RCHAN_SB_CLEAR_NOREF(x) \ | |
34 | (x = (struct chanbuf_page *)((unsigned long)(x) & ~RCHAN_NOREF_FLAG)) | |
35 | ||
36 | struct ltt_trace; | |
37 | ||
38 | struct chanbuf_page { | |
39 | void *virt; /* page virtual address (cached) */ | |
40 | struct page *page; /* pointer to page structure */ | |
41 | }; | |
42 | ||
43 | struct chanbuf_sb { | |
44 | struct chanbuf_page *pages; /* Pointer to rchan pages for subbuf */ | |
45 | }; | |
46 | ||
47 | struct ltt_chanbuf_alloc { | |
48 | struct chanbuf_sb *buf_wsb; /* Array of rchan_sb for writer */ | |
49 | struct chanbuf_sb buf_rsb; /* chanbuf_sb for reader */ | |
50 | void **_virt; /* Array of pointers to page addr */ | |
51 | struct page **_pages; /* Array of pointers to pages */ | |
52 | struct dentry *dentry; /* Associated file dentry */ | |
53 | unsigned int nr_pages; /* Number pages in buffer */ | |
54 | ||
55 | struct ltt_chan_alloc *chan; /* Associated channel */ | |
56 | unsigned int cpu; /* This buffer's cpu */ | |
57 | unsigned int allocated:1; /* Bool: is buffer allocated ? */ | |
58 | }; | |
59 | ||
60 | int ltt_chanbuf_alloc_create(struct ltt_chanbuf_alloc *buf, | |
61 | struct ltt_chan_alloc *chan, int cpu); | |
62 | void ltt_chanbuf_alloc_free(struct ltt_chanbuf_alloc *buf); | |
63 | int ltt_chan_alloc_init(struct ltt_chan_alloc *chan, struct ltt_trace *trace, | |
64 | const char *base_filename, | |
65 | struct dentry *parent, size_t sb_size, | |
66 | size_t n_sb, int extra_reader_sb, int overwrite); | |
67 | void ltt_chan_alloc_free(struct ltt_chan_alloc *chan); | |
68 | void ltt_chan_alloc_remove_files(struct ltt_chan_alloc *chan); | |
69 | int ltt_chanbuf_create_file(const char *filename, struct dentry *parent, | |
70 | int mode, struct ltt_chanbuf *buf); | |
71 | int ltt_chanbuf_remove_file(struct ltt_chanbuf *buf); | |
72 | ||
73 | void ltt_chan_for_each_channel(void (*cb) (struct ltt_chanbuf *buf), int cpu); | |
74 | ||
75 | extern void _ltt_relay_write(struct ltt_chanbuf_alloc *bufa, | |
76 | size_t offset, const void *src, size_t len, | |
77 | ssize_t pagecpy); | |
78 | ||
79 | extern void _ltt_relay_strncpy(struct ltt_chanbuf_alloc *bufa, | |
80 | size_t offset, const void *src, size_t len, | |
81 | ssize_t pagecpy); | |
82 | ||
83 | extern void _ltt_relay_strncpy_fixup(struct ltt_chanbuf_alloc *bufa, | |
84 | size_t offset, size_t len, size_t copied, | |
85 | int terminated); | |
86 | ||
87 | extern int ltt_relay_read(struct ltt_chanbuf_alloc *bufa, | |
88 | size_t offset, void *dest, size_t len); | |
89 | ||
90 | extern int ltt_relay_read_cstr(struct ltt_chanbuf_alloc *bufa, | |
91 | size_t offset, void *dest, size_t len); | |
92 | ||
93 | extern struct page *ltt_relay_read_get_page(struct ltt_chanbuf_alloc *bufa, | |
94 | size_t offset); | |
95 | ||
96 | /* | |
97 | * Return the address where a given offset is located. | |
98 | * Should be used to get the current subbuffer header pointer. Given we know | |
99 | * it's never on a page boundary, it's safe to write directly to this address, | |
100 | * as long as the write is never bigger than a page size. | |
101 | */ | |
102 | extern void *ltt_relay_offset_address(struct ltt_chanbuf_alloc *bufa, | |
103 | size_t offset); | |
104 | extern void *ltt_relay_read_offset_address(struct ltt_chanbuf_alloc *bufa, | |
105 | size_t offset); | |
106 | ||
107 | #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS | |
108 | static __inline__ | |
109 | void ltt_relay_do_copy(void *dest, const void *src, size_t len) | |
110 | { | |
111 | switch (len) { | |
112 | case 0: | |
113 | break; | |
114 | case 1: | |
115 | *(u8 *)dest = *(const u8 *)src; | |
116 | break; | |
117 | case 2: | |
118 | *(u16 *)dest = *(const u16 *)src; | |
119 | break; | |
120 | case 4: | |
121 | *(u32 *)dest = *(const u32 *)src; | |
122 | break; | |
123 | case 8: | |
124 | *(u64 *)dest = *(const u64 *)src; | |
125 | break; | |
126 | default: | |
127 | /* | |
128 | * What we really want here is an __inline__ memcpy, but we don't | |
129 | * have constants, so gcc generally uses a function call. | |
130 | */ | |
131 | for (; len > 0; len--) | |
132 | *(u8 *)dest++ = *(const u8 *)src++; | |
133 | } | |
134 | } | |
135 | #else | |
136 | /* | |
137 | * Returns whether the dest and src addresses are aligned on | |
138 | * min(sizeof(void *), len). Call this with statically known len for efficiency. | |
139 | */ | |
140 | static __inline__ | |
141 | int addr_aligned(const void *dest, const void *src, size_t len) | |
142 | { | |
143 | if (ltt_align((size_t)dest, len)) | |
144 | return 0; | |
145 | if (ltt_align((size_t)src, len)) | |
146 | return 0; | |
147 | return 1; | |
148 | } | |
149 | ||
150 | static __inline__ | |
151 | void ltt_relay_do_copy(void *dest, const void *src, size_t len) | |
152 | { | |
153 | switch (len) { | |
154 | case 0: | |
155 | break; | |
156 | case 1: | |
157 | *(u8 *)dest = *(const u8 *)src; | |
158 | break; | |
159 | case 2: | |
160 | if (unlikely(!addr_aligned(dest, src, 2))) | |
161 | goto memcpy_fallback; | |
162 | *(u16 *)dest = *(const u16 *)src; | |
163 | break; | |
164 | case 4: | |
165 | if (unlikely(!addr_aligned(dest, src, 4))) | |
166 | goto memcpy_fallback; | |
167 | *(u32 *)dest = *(const u32 *)src; | |
168 | break; | |
169 | case 8: | |
170 | if (unlikely(!addr_aligned(dest, src, 8))) | |
171 | goto memcpy_fallback; | |
172 | *(u64 *)dest = *(const u64 *)src; | |
173 | break; | |
174 | default: | |
175 | goto memcpy_fallback; | |
176 | } | |
177 | return; | |
178 | ||
179 | memcpy_fallback: | |
180 | /* | |
181 | * What we really want here is an inline memcpy, but we don't | |
182 | * have constants, so gcc generally uses a function call. | |
183 | */ | |
184 | for (; len > 0; len--) | |
185 | *(u8 *)dest++ = *(const u8 *)src++; | |
186 | } | |
187 | #endif | |
188 | ||
189 | /* | |
190 | * ltt_relay_do_memset - write character into dest. | |
191 | * @dest: destination | |
192 | * @src: source character | |
193 | * @len: length to write | |
194 | */ | |
195 | static __inline__ | |
196 | void ltt_relay_do_memset(void *dest, char src, size_t len) | |
197 | { | |
198 | /* | |
199 | * What we really want here is an __inline__ memset, but we | |
200 | * don't have constants, so gcc generally uses a function call. | |
201 | */ | |
202 | for (; len > 0; len--) | |
203 | *(u8 *)dest++ = src; | |
204 | } | |
205 | ||
206 | ||
207 | /* | |
208 | * ltt_relay_do_strncpy - copy a string up to a certain number of bytes | |
209 | * @dest: destination | |
210 | * @src: source | |
211 | * @len: max. length to copy | |
212 | * @terminated: output string ends with \0 (output) | |
213 | * | |
214 | * returns the number of bytes copied. Does not finalize with \0 if len is | |
215 | * reached. | |
216 | */ | |
217 | static __inline__ | |
218 | size_t ltt_relay_do_strncpy(void *dest, const void *src, size_t len, | |
219 | int *terminated) | |
220 | { | |
221 | size_t orig_len = len; | |
222 | ||
223 | *terminated = 0; | |
224 | /* | |
225 | * What we really want here is an __inline__ strncpy, but we | |
226 | * don't have constants, so gcc generally uses a function call. | |
227 | */ | |
228 | for (; len > 0; len--) { | |
229 | *(u8 *)dest = ACCESS_ONCE(*(const u8 *)src); | |
230 | /* Check with dest, because src may be modified concurrently */ | |
231 | if (*(const u8 *)dest == '\0') { | |
232 | len--; | |
233 | *terminated = 1; | |
234 | break; | |
235 | } | |
236 | dest++; | |
237 | src++; | |
238 | } | |
239 | return orig_len - len; | |
240 | } | |
241 | ||
242 | static __inline__ | |
243 | int ltt_relay_write(struct ltt_chanbuf_alloc *bufa, | |
244 | struct ltt_chan_alloc *chana, size_t offset, | |
245 | const void *src, size_t len) | |
246 | { | |
247 | size_t sbidx, index; | |
248 | ssize_t pagecpy; | |
249 | struct chanbuf_page *rpages; | |
250 | ||
251 | offset &= chana->buf_size - 1; | |
252 | sbidx = offset >> chana->sb_size_order; | |
253 | index = (offset & (chana->sb_size - 1)) >> PAGE_SHIFT; | |
254 | pagecpy = min_t(size_t, len, (- offset) & ~PAGE_MASK); | |
255 | rpages = bufa->buf_wsb[sbidx].pages; | |
256 | WARN_ON_ONCE(RCHAN_SB_IS_NOREF(rpages)); | |
257 | ltt_relay_do_copy(rpages[index].virt + (offset & ~PAGE_MASK), | |
258 | src, pagecpy); | |
259 | ||
260 | if (unlikely(len != pagecpy)) | |
261 | _ltt_relay_write(bufa, offset, src, len, pagecpy); | |
262 | return len; | |
263 | } | |
264 | ||
265 | static __inline__ | |
266 | int ltt_relay_strncpy(struct ltt_chanbuf_alloc *bufa, | |
267 | struct ltt_chan_alloc *chana, size_t offset, | |
268 | const void *src, size_t len) | |
269 | { | |
270 | size_t sbidx, index; | |
271 | ssize_t pagecpy, copied; | |
272 | struct chanbuf_page *rpages; | |
273 | int terminated; | |
274 | ||
275 | offset &= chana->buf_size - 1; | |
276 | sbidx = offset >> chana->sb_size_order; | |
277 | index = (offset & (chana->sb_size - 1)) >> PAGE_SHIFT; | |
278 | pagecpy = min_t(size_t, len, (- offset) & ~PAGE_MASK); | |
279 | rpages = bufa->buf_wsb[sbidx].pages; | |
280 | WARN_ON_ONCE(RCHAN_SB_IS_NOREF(rpages)); | |
281 | copied = ltt_relay_do_strncpy(rpages[index].virt | |
282 | + (offset & ~PAGE_MASK), | |
283 | src, pagecpy, &terminated); | |
284 | if (unlikely(copied < pagecpy || ((len == pagecpy) && !terminated))) | |
285 | _ltt_relay_strncpy_fixup(bufa, offset, len, copied, | |
286 | terminated); | |
287 | else { | |
288 | if (unlikely(len != pagecpy)) | |
289 | _ltt_relay_strncpy(bufa, offset, src, len, pagecpy); | |
290 | } | |
291 | return len; | |
292 | } | |
293 | ||
294 | /** | |
295 | * ltt_clear_noref_flag - Clear the noref subbuffer flag, for writer. | |
296 | */ | |
297 | static __inline__ | |
298 | void ltt_clear_noref_flag(struct ltt_chanbuf_alloc *bufa, long idx) | |
299 | { | |
300 | struct chanbuf_page *sb_pages, *new_sb_pages; | |
301 | ||
302 | sb_pages = bufa->buf_wsb[idx].pages; | |
303 | for (;;) { | |
304 | if (!RCHAN_SB_IS_NOREF(sb_pages)) | |
305 | return; /* Already writing to this buffer */ | |
306 | new_sb_pages = sb_pages; | |
307 | RCHAN_SB_CLEAR_NOREF(new_sb_pages); | |
308 | new_sb_pages = cmpxchg(&bufa->buf_wsb[idx].pages, | |
309 | sb_pages, new_sb_pages); | |
310 | if (likely(new_sb_pages == sb_pages)) | |
311 | break; | |
312 | sb_pages = new_sb_pages; | |
313 | } | |
314 | } | |
315 | ||
316 | /** | |
317 | * ltt_set_noref_flag - Set the noref subbuffer flag, for writer. | |
318 | */ | |
319 | static __inline__ | |
320 | void ltt_set_noref_flag(struct ltt_chanbuf_alloc *bufa, long idx) | |
321 | { | |
322 | struct chanbuf_page *sb_pages, *new_sb_pages; | |
323 | ||
324 | sb_pages = bufa->buf_wsb[idx].pages; | |
325 | for (;;) { | |
326 | if (RCHAN_SB_IS_NOREF(sb_pages)) | |
327 | return; /* Already set */ | |
328 | new_sb_pages = sb_pages; | |
329 | RCHAN_SB_SET_NOREF(new_sb_pages); | |
330 | new_sb_pages = cmpxchg(&bufa->buf_wsb[idx].pages, | |
331 | sb_pages, new_sb_pages); | |
332 | if (likely(new_sb_pages == sb_pages)) | |
333 | break; | |
334 | sb_pages = new_sb_pages; | |
335 | } | |
336 | } | |
337 | ||
338 | /** | |
339 | * update_read_sb_index - Read-side subbuffer index update. | |
340 | */ | |
341 | static __inline__ | |
342 | int update_read_sb_index(struct ltt_chanbuf_alloc *bufa, | |
343 | struct ltt_chan_alloc *chana, | |
344 | long consumed_idx) | |
345 | { | |
346 | struct chanbuf_page *old_wpage, *new_wpage; | |
347 | ||
348 | if (unlikely(chana->extra_reader_sb)) { | |
349 | /* | |
350 | * Exchange the target writer subbuffer with our own unused | |
351 | * subbuffer. | |
352 | */ | |
353 | old_wpage = bufa->buf_wsb[consumed_idx].pages; | |
354 | if (unlikely(!RCHAN_SB_IS_NOREF(old_wpage))) | |
355 | return -EAGAIN; | |
356 | WARN_ON_ONCE(!RCHAN_SB_IS_NOREF(bufa->buf_rsb.pages)); | |
357 | new_wpage = cmpxchg(&bufa->buf_wsb[consumed_idx].pages, | |
358 | old_wpage, | |
359 | bufa->buf_rsb.pages); | |
360 | if (unlikely(old_wpage != new_wpage)) | |
361 | return -EAGAIN; | |
362 | bufa->buf_rsb.pages = new_wpage; | |
363 | RCHAN_SB_CLEAR_NOREF(bufa->buf_rsb.pages); | |
364 | } else { | |
365 | /* No page exchange, use the writer page directly */ | |
366 | bufa->buf_rsb.pages = bufa->buf_wsb[consumed_idx].pages; | |
367 | RCHAN_SB_CLEAR_NOREF(bufa->buf_rsb.pages); | |
368 | } | |
369 | return 0; | |
370 | } | |
371 | ||
372 | ssize_t ltt_relay_file_splice_read(struct file *in, loff_t *ppos, | |
373 | struct pipe_inode_info *pipe, size_t len, | |
374 | unsigned int flags); | |
375 | loff_t ltt_relay_no_llseek(struct file *file, loff_t offset, int origin); | |
376 | ||
13c60117 MD |
377 | extern int ltt_ascii_init(void); |
378 | extern void ltt_ascii_exit(void); | |
379 | ||
1c8284eb | 380 | #endif /* _LINUX_LTT_RELAY_H */ |