Commit | Line | Data |
---|---|---|
b5b073e2 PMF |
1 | /* |
2 | * buffers.h | |
3 | * | |
4 | * Copyright (C) 2009 - Pierre-Marc Fournier (pierre-marc dot fournier at polymtl dot ca) | |
5 | * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca) | |
6 | * | |
7 | */ | |
8 | ||
9 | #ifndef _UST_BUFFERS_H | |
10 | #define _UST_BUFFERS_H | |
11 | ||
12 | #include <kcompat/kref.h> | |
13 | #include <assert.h> | |
14 | #include "channels.h" | |
15 | #include "buffers.h" | |
16 | ||
17 | /* Return the size of the minimum number of pages that can contain x. */ | |
18 | #define FIX_SIZE(x) ((((x) - 1) & PAGE_MASK) + PAGE_SIZE) | |
19 | ||
20 | /* | |
21 | * BUFFER_TRUNC zeroes the subbuffer offset and the subbuffer number parts of | |
22 | * the offset, which leaves only the buffer number. | |
23 | */ | |
24 | #define BUFFER_TRUNC(offset, chan) \ | |
25 | ((offset) & (~((chan)->alloc_size-1))) | |
26 | #define BUFFER_OFFSET(offset, chan) ((offset) & ((chan)->alloc_size - 1)) | |
27 | #define SUBBUF_OFFSET(offset, chan) ((offset) & ((chan)->subbuf_size - 1)) | |
28 | #define SUBBUF_ALIGN(offset, chan) \ | |
29 | (((offset) + (chan)->subbuf_size) & (~((chan)->subbuf_size - 1))) | |
30 | #define SUBBUF_TRUNC(offset, chan) \ | |
31 | ((offset) & (~((chan)->subbuf_size - 1))) | |
32 | #define SUBBUF_INDEX(offset, chan) \ | |
33 | (BUFFER_OFFSET((offset), chan) >> (chan)->subbuf_size_order) | |
34 | ||
35 | /* | |
36 | * Tracks changes to rchan/rchan_buf structs | |
37 | */ | |
38 | #define UST_CHANNEL_VERSION 8 | |
39 | ||
40 | struct ust_buffer { | |
41 | /* First 32 bytes cache-hot cacheline */ | |
42 | local_t offset; /* Current offset in the buffer */ | |
43 | local_t *commit_count; /* Commit count per sub-buffer */ | |
44 | atomic_long_t consumed; /* | |
45 | * Current offset in the buffer | |
46 | * standard atomic access (shared) | |
47 | */ | |
48 | unsigned long last_tsc; /* | |
49 | * Last timestamp written in the buffer. | |
50 | */ | |
51 | /* End of first 32 bytes cacheline */ | |
52 | atomic_long_t active_readers; /* | |
53 | * Active readers count | |
54 | * standard atomic access (shared) | |
55 | */ | |
56 | local_t events_lost; | |
57 | local_t corrupted_subbuffers; | |
58 | /* one byte is written to this pipe when data is available, in order | |
59 | to wake the consumer */ | |
60 | /* portability: Single byte writes must be as quick as possible. The kernel-side | |
61 | buffer must be large enough so the writer doesn't block. From the pipe(7) | |
62 | man page: Since linux 2.6.11, the pipe capacity is 65536 bytes. */ | |
63 | int data_ready_fd_write; | |
64 | /* the reading end of the pipe */ | |
65 | int data_ready_fd_read; | |
66 | ||
67 | struct ust_channel *chan; | |
68 | struct kref kref; | |
69 | void *buf_data; | |
70 | size_t buf_size; | |
71 | int shmid; | |
204141ee | 72 | unsigned int cpu; |
b5b073e2 PMF |
73 | |
74 | /* commit count per subbuffer; must be at end of struct */ | |
75 | local_t commit_seq[0] ____cacheline_aligned; | |
76 | } ____cacheline_aligned; | |
77 | ||
78 | extern void _ust_buffers_write(struct ust_buffer *buf, size_t offset, | |
79 | const void *src, size_t len, ssize_t cpy); | |
80 | ||
81 | /* | |
82 | * Return the address where a given offset is located. | |
83 | * Should be used to get the current subbuffer header pointer. Given we know | |
84 | * it's never on a page boundary, it's safe to write directly to this address, | |
85 | * as long as the write is never bigger than a page size. | |
86 | */ | |
87 | extern void *ust_buffers_offset_address(struct ust_buffer *buf, | |
88 | size_t offset); | |
89 | ||
90 | /* FIXME: lttng has a version for systems with inefficient unaligned access */ | |
a2fd50ef | 91 | static __inline__ void ust_buffers_do_copy(void *dest, const void *src, size_t len) |
b5b073e2 PMF |
92 | { |
93 | union { | |
94 | const void *src; | |
95 | const u8 *src8; | |
96 | const u16 *src16; | |
97 | const u32 *src32; | |
98 | const u64 *src64; | |
99 | } u = { .src = src }; | |
100 | ||
101 | switch (len) { | |
102 | case 0: break; | |
103 | case 1: *(u8 *)dest = *u.src8; | |
104 | break; | |
105 | case 2: *(u16 *)dest = *u.src16; | |
106 | break; | |
107 | case 4: *(u32 *)dest = *u.src32; | |
108 | break; | |
109 | case 8: *(u64 *)dest = *u.src64; | |
110 | break; | |
111 | default: | |
112 | memcpy(dest, src, len); | |
113 | } | |
114 | } | |
115 | ||
116 | /* FIXME: there is both a static inline and a '_' non static inline version ?? */ | |
a2fd50ef | 117 | static __inline__ int ust_buffers_write(struct ust_buffer *buf, size_t offset, |
b5b073e2 PMF |
118 | const void *src, size_t len) |
119 | { | |
120 | size_t cpy; | |
121 | size_t buf_offset = BUFFER_OFFSET(offset, buf->chan); | |
122 | ||
123 | assert(buf_offset < buf->chan->subbuf_size*buf->chan->subbuf_cnt); | |
124 | ||
125 | cpy = min_t(size_t, len, buf->buf_size - buf_offset); | |
126 | ust_buffers_do_copy(buf->buf_data + buf_offset, src, cpy); | |
127 | ||
128 | if (unlikely(len != cpy)) | |
129 | _ust_buffers_write(buf, buf_offset, src, len, cpy); | |
130 | return len; | |
131 | } | |
132 | ||
133 | int ust_buffers_channel_open(struct ust_channel *chan, size_t subbuf_size, size_t n_subbufs); | |
134 | extern void ust_buffers_channel_close(struct ust_channel *chan); | |
135 | ||
136 | extern int ust_buffers_do_get_subbuf(struct ust_buffer *buf, long *pconsumed_old); | |
137 | ||
138 | extern int ust_buffers_do_put_subbuf(struct ust_buffer *buf, u32 uconsumed_old); | |
139 | ||
140 | extern void init_ustrelay_transport(void); | |
141 | ||
142 | /*static*/ /* inline */ notrace void ltt_commit_slot( | |
143 | struct ust_channel *channel, | |
144 | void **transport_data, long buf_offset, | |
145 | size_t data_size, size_t slot_size); | |
146 | ||
147 | #endif /* _UST_BUFFERS_H */ |