Commit | Line | Data |
---|---|---|
b7cdc182 | 1 | /* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) |
9f36eaed | 2 | * |
24591303 | 3 | * ringbuffer/frontend_internal.h |
f3bc08c5 | 4 | * |
f3bc08c5 MD |
5 | * Ring Buffer Library Synchronization Header (internal helpers). |
6 | * | |
886d51a3 MD |
7 | * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
8 | * | |
f3bc08c5 | 9 | * See ring_buffer_frontend.c for more information on wait-free algorithms. |
f3bc08c5 MD |
10 | */ |
11 | ||
9f36eaed MJ |
12 | #ifndef _LIB_RING_BUFFER_FRONTEND_INTERNAL_H |
13 | #define _LIB_RING_BUFFER_FRONTEND_INTERNAL_H | |
14 | ||
24591303 MD |
15 | #include <ringbuffer/config.h> |
16 | #include <ringbuffer/backend_types.h> | |
17 | #include <ringbuffer/frontend_types.h> | |
a071f25d | 18 | #include <lttng/prio_heap.h> /* For per-CPU read-side iterator */ |
f3bc08c5 MD |
19 | |
20 | /* Buffer offset macros */ | |
21 | ||
22 | /* buf_trunc mask selects only the buffer number. */ | |
23 | static inline | |
860c213b | 24 | unsigned long buf_trunc(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
25 | { |
26 | return offset & ~(chan->backend.buf_size - 1); | |
27 | ||
28 | } | |
29 | ||
30 | /* Select the buffer number value (counter). */ | |
31 | static inline | |
860c213b | 32 | unsigned long buf_trunc_val(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
33 | { |
34 | return buf_trunc(offset, chan) >> chan->backend.buf_size_order; | |
35 | } | |
36 | ||
37 | /* buf_offset mask selects only the offset within the current buffer. */ | |
38 | static inline | |
860c213b | 39 | unsigned long buf_offset(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
40 | { |
41 | return offset & (chan->backend.buf_size - 1); | |
42 | } | |
43 | ||
44 | /* subbuf_offset mask selects the offset within the current subbuffer. */ | |
45 | static inline | |
860c213b | 46 | unsigned long subbuf_offset(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
47 | { |
48 | return offset & (chan->backend.subbuf_size - 1); | |
49 | } | |
50 | ||
51 | /* subbuf_trunc mask selects the subbuffer number. */ | |
52 | static inline | |
860c213b | 53 | unsigned long subbuf_trunc(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
54 | { |
55 | return offset & ~(chan->backend.subbuf_size - 1); | |
56 | } | |
57 | ||
58 | /* subbuf_align aligns the offset to the next subbuffer. */ | |
59 | static inline | |
860c213b | 60 | unsigned long subbuf_align(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
61 | { |
62 | return (offset + chan->backend.subbuf_size) | |
63 | & ~(chan->backend.subbuf_size - 1); | |
64 | } | |
65 | ||
66 | /* subbuf_index returns the index of the current subbuffer within the buffer. */ | |
67 | static inline | |
860c213b | 68 | unsigned long subbuf_index(unsigned long offset, struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
69 | { |
70 | return buf_offset(offset, chan) >> chan->backend.subbuf_size_order; | |
71 | } | |
72 | ||
73 | /* | |
3a6d0934 MD |
74 | * Last timestamp comparison functions. Check if the current timestamp |
75 | * overflows timestamp_bits bits from the last timestamp read. When | |
76 | * overflows are detected, the full 64-bit timestamp counter should be | |
77 | * written in the record header. Reads and writes last_timestamp | |
78 | * atomically. | |
f3bc08c5 MD |
79 | */ |
80 | ||
81 | #if (BITS_PER_LONG == 32) | |
82 | static inline | |
3a6d0934 MD |
83 | void save_last_timestamp(const struct lttng_kernel_ring_buffer_config *config, |
84 | struct lttng_kernel_ring_buffer *buf, u64 timestamp) | |
f3bc08c5 | 85 | { |
3a6d0934 | 86 | if (config->timestamp_bits == 0 || config->timestamp_bits == 64) |
f3bc08c5 MD |
87 | return; |
88 | ||
89 | /* | |
90 | * Ensure the compiler performs this update in a single instruction. | |
91 | */ | |
3a6d0934 | 92 | v_set(config, &buf->last_timestamp, (unsigned long)(timestamp >> config->timestamp_bits)); |
f3bc08c5 MD |
93 | } |
94 | ||
95 | static inline | |
3a6d0934 MD |
96 | int last_timestamp_overflow(const struct lttng_kernel_ring_buffer_config *config, |
97 | struct lttng_kernel_ring_buffer *buf, u64 timestamp) | |
f3bc08c5 | 98 | { |
3a6d0934 | 99 | unsigned long timestamp_shifted; |
f3bc08c5 | 100 | |
3a6d0934 | 101 | if (config->timestamp_bits == 0 || config->timestamp_bits == 64) |
f3bc08c5 MD |
102 | return 0; |
103 | ||
3a6d0934 MD |
104 | timestamp_shifted = (unsigned long)(timestamp >> config->timestamp_bits); |
105 | if (unlikely(timestamp_shifted | |
106 | - (unsigned long)v_read(config, &buf->last_timestamp))) | |
f3bc08c5 MD |
107 | return 1; |
108 | else | |
109 | return 0; | |
110 | } | |
111 | #else | |
112 | static inline | |
3a6d0934 MD |
113 | void save_last_timestamp(const struct lttng_kernel_ring_buffer_config *config, |
114 | struct lttng_kernel_ring_buffer *buf, u64 timestamp) | |
f3bc08c5 | 115 | { |
3a6d0934 | 116 | if (config->timestamp_bits == 0 || config->timestamp_bits == 64) |
f3bc08c5 MD |
117 | return; |
118 | ||
3a6d0934 | 119 | v_set(config, &buf->last_timestamp, (unsigned long)timestamp); |
f3bc08c5 MD |
120 | } |
121 | ||
122 | static inline | |
3a6d0934 MD |
123 | int last_timestamp_overflow(const struct lttng_kernel_ring_buffer_config *config, |
124 | struct lttng_kernel_ring_buffer *buf, u64 timestamp) | |
f3bc08c5 | 125 | { |
3a6d0934 | 126 | if (config->timestamp_bits == 0 || config->timestamp_bits == 64) |
f3bc08c5 MD |
127 | return 0; |
128 | ||
3a6d0934 MD |
129 | if (unlikely((timestamp - v_read(config, &buf->last_timestamp)) |
130 | >> config->timestamp_bits)) | |
f3bc08c5 MD |
131 | return 1; |
132 | else | |
133 | return 0; | |
134 | } | |
135 | #endif | |
136 | ||
137 | extern | |
8a57ec02 | 138 | int lib_ring_buffer_reserve_slow(struct lttng_kernel_ring_buffer_ctx *ctx, |
cc62f29e | 139 | void *client_ctx); |
f3bc08c5 MD |
140 | |
141 | extern | |
e20c0fec | 142 | void lib_ring_buffer_switch_slow(struct lttng_kernel_ring_buffer *buf, |
f3bc08c5 MD |
143 | enum switch_mode mode); |
144 | ||
aece661f | 145 | extern |
e20c0fec MD |
146 | void lib_ring_buffer_check_deliver_slow(const struct lttng_kernel_ring_buffer_config *config, |
147 | struct lttng_kernel_ring_buffer *buf, | |
860c213b | 148 | struct lttng_kernel_ring_buffer_channel *chan, |
aece661f MD |
149 | unsigned long offset, |
150 | unsigned long commit_count, | |
151 | unsigned long idx, | |
b2cf5e0b | 152 | const struct lttng_kernel_ring_buffer_ctx *ctx); |
aece661f | 153 | |
5e391252 | 154 | extern |
e20c0fec | 155 | void lib_ring_buffer_switch_remote(struct lttng_kernel_ring_buffer *buf); |
c6f05468 | 156 | extern |
e20c0fec | 157 | void lib_ring_buffer_switch_remote_empty(struct lttng_kernel_ring_buffer *buf); |
c245d0d3 | 158 | extern |
e20c0fec | 159 | void lib_ring_buffer_clear(struct lttng_kernel_ring_buffer *buf); |
5e391252 | 160 | |
f3bc08c5 MD |
161 | /* Buffer write helpers */ |
162 | ||
163 | static inline | |
e20c0fec | 164 | void lib_ring_buffer_reserve_push_reader(struct lttng_kernel_ring_buffer *buf, |
860c213b | 165 | struct lttng_kernel_ring_buffer_channel *chan, |
f3bc08c5 MD |
166 | unsigned long offset) |
167 | { | |
168 | unsigned long consumed_old, consumed_new; | |
169 | ||
170 | do { | |
171 | consumed_old = atomic_long_read(&buf->consumed); | |
172 | /* | |
173 | * If buffer is in overwrite mode, push the reader consumed | |
174 | * count if the write position has reached it and we are not | |
175 | * at the first iteration (don't push the reader farther than | |
176 | * the writer). This operation can be done concurrently by many | |
177 | * writers in the same buffer, the writer being at the farthest | |
178 | * write position sub-buffer index in the buffer being the one | |
179 | * which will win this loop. | |
180 | */ | |
181 | if (unlikely(subbuf_trunc(offset, chan) | |
182 | - subbuf_trunc(consumed_old, chan) | |
183 | >= chan->backend.buf_size)) | |
184 | consumed_new = subbuf_align(consumed_old, chan); | |
185 | else | |
186 | return; | |
187 | } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, | |
188 | consumed_new) != consumed_old)); | |
189 | } | |
190 | ||
c245d0d3 MD |
191 | /* |
192 | * Move consumed position to the beginning of subbuffer in which the | |
ac4a87e5 MD |
193 | * write offset is. Should only be used on ring buffers that are not |
194 | * actively being written into, because clear_reader does not take into | |
195 | * account the commit counters when moving the consumed position, which | |
196 | * can make concurrent trace producers or consumers observe consumed | |
197 | * position further than the write offset, which breaks ring buffer | |
198 | * algorithm guarantees. | |
c245d0d3 MD |
199 | */ |
200 | static inline | |
e20c0fec | 201 | void lib_ring_buffer_clear_reader(struct lttng_kernel_ring_buffer *buf, |
860c213b | 202 | struct lttng_kernel_ring_buffer_channel *chan) |
c245d0d3 | 203 | { |
e20c0fec | 204 | const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config; |
c245d0d3 MD |
205 | unsigned long offset, consumed_old, consumed_new; |
206 | ||
207 | do { | |
208 | offset = v_read(config, &buf->offset); | |
209 | consumed_old = atomic_long_read(&buf->consumed); | |
ac4a87e5 MD |
210 | CHAN_WARN_ON(chan, (long) (subbuf_trunc(offset, chan) |
211 | - subbuf_trunc(consumed_old, chan)) | |
212 | < 0); | |
213 | consumed_new = subbuf_trunc(offset, chan); | |
c245d0d3 MD |
214 | } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, |
215 | consumed_new) != consumed_old)); | |
216 | } | |
217 | ||
f3bc08c5 | 218 | static inline |
e20c0fec MD |
219 | int lib_ring_buffer_pending_data(const struct lttng_kernel_ring_buffer_config *config, |
220 | struct lttng_kernel_ring_buffer *buf, | |
860c213b | 221 | struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
222 | { |
223 | return !!subbuf_offset(v_read(config, &buf->offset), chan); | |
224 | } | |
225 | ||
226 | static inline | |
e20c0fec MD |
227 | unsigned long lib_ring_buffer_get_data_size(const struct lttng_kernel_ring_buffer_config *config, |
228 | struct lttng_kernel_ring_buffer *buf, | |
f3bc08c5 MD |
229 | unsigned long idx) |
230 | { | |
231 | return subbuffer_get_data_size(config, &buf->backend, idx); | |
232 | } | |
233 | ||
234 | /* | |
235 | * Check if all space reservation in a buffer have been committed. This helps | |
236 | * knowing if an execution context is nested (for per-cpu buffers only). | |
237 | * This is a very specific ftrace use-case, so we keep this as "internal" API. | |
238 | */ | |
239 | static inline | |
e20c0fec MD |
240 | int lib_ring_buffer_reserve_committed(const struct lttng_kernel_ring_buffer_config *config, |
241 | struct lttng_kernel_ring_buffer *buf, | |
860c213b | 242 | struct lttng_kernel_ring_buffer_channel *chan) |
f3bc08c5 MD |
243 | { |
244 | unsigned long offset, idx, commit_count; | |
245 | ||
246 | CHAN_WARN_ON(chan, config->alloc != RING_BUFFER_ALLOC_PER_CPU); | |
247 | CHAN_WARN_ON(chan, config->sync != RING_BUFFER_SYNC_PER_CPU); | |
248 | ||
249 | /* | |
250 | * Read offset and commit count in a loop so they are both read | |
251 | * atomically wrt interrupts. By deal with interrupt concurrency by | |
252 | * restarting both reads if the offset has been pushed. Note that given | |
253 | * we only have to deal with interrupt concurrency here, an interrupt | |
254 | * modifying the commit count will also modify "offset", so it is safe | |
255 | * to only check for offset modifications. | |
256 | */ | |
257 | do { | |
258 | offset = v_read(config, &buf->offset); | |
259 | idx = subbuf_index(offset, chan); | |
260 | commit_count = v_read(config, &buf->commit_hot[idx].cc); | |
261 | } while (offset != v_read(config, &buf->offset)); | |
262 | ||
263 | return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) | |
264 | - (commit_count & chan->commit_count_mask) == 0); | |
265 | } | |
266 | ||
635e457c | 267 | /* |
3a6d0934 | 268 | * Receive end of subbuffer timestamp as parameter. It has been read in the |
635e457c MD |
269 | * space reservation loop of either reserve or switch, which ensures it |
270 | * progresses monotonically with event records in the buffer. Therefore, | |
271 | * it ensures that the end timestamp of a subbuffer is <= begin | |
272 | * timestamp of the following subbuffers. | |
273 | */ | |
f3bc08c5 | 274 | static inline |
e20c0fec MD |
275 | void lib_ring_buffer_check_deliver(const struct lttng_kernel_ring_buffer_config *config, |
276 | struct lttng_kernel_ring_buffer *buf, | |
860c213b | 277 | struct lttng_kernel_ring_buffer_channel *chan, |
f3bc08c5 MD |
278 | unsigned long offset, |
279 | unsigned long commit_count, | |
635e457c | 280 | unsigned long idx, |
b2cf5e0b | 281 | const struct lttng_kernel_ring_buffer_ctx *ctx) |
f3bc08c5 MD |
282 | { |
283 | unsigned long old_commit_count = commit_count | |
284 | - chan->backend.subbuf_size; | |
f3bc08c5 MD |
285 | |
286 | /* Check if all commits have been done */ | |
287 | if (unlikely((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) | |
aece661f MD |
288 | - (old_commit_count & chan->commit_count_mask) == 0)) |
289 | lib_ring_buffer_check_deliver_slow(config, buf, chan, offset, | |
b2cf5e0b | 290 | commit_count, idx, ctx); |
f3bc08c5 MD |
291 | } |
292 | ||
293 | /* | |
294 | * lib_ring_buffer_write_commit_counter | |
295 | * | |
296 | * For flight recording. must be called after commit. | |
297 | * This function increments the subbuffer's commit_seq counter each time the | |
298 | * commit count reaches back the reserve offset (modulo subbuffer size). It is | |
299 | * useful for crash dump. | |
300 | */ | |
301 | static inline | |
e20c0fec MD |
302 | void lib_ring_buffer_write_commit_counter(const struct lttng_kernel_ring_buffer_config *config, |
303 | struct lttng_kernel_ring_buffer *buf, | |
860c213b | 304 | struct lttng_kernel_ring_buffer_channel *chan, |
f3bc08c5 | 305 | unsigned long buf_offset, |
8ec496cf MD |
306 | unsigned long commit_count, |
307 | struct commit_counters_hot *cc_hot) | |
f3bc08c5 | 308 | { |
7915e163 | 309 | unsigned long commit_seq_old; |
f3bc08c5 MD |
310 | |
311 | if (config->oops != RING_BUFFER_OOPS_CONSISTENCY) | |
312 | return; | |
313 | ||
f3bc08c5 MD |
314 | /* |
315 | * subbuf_offset includes commit_count_mask. We can simply | |
316 | * compare the offsets within the subbuffer without caring about | |
317 | * buffer full/empty mismatch because offset is never zero here | |
318 | * (subbuffer header and record headers have non-zero length). | |
319 | */ | |
7915e163 | 320 | if (unlikely(subbuf_offset(buf_offset - commit_count, chan))) |
f3bc08c5 MD |
321 | return; |
322 | ||
8ec496cf | 323 | commit_seq_old = v_read(config, &cc_hot->seq); |
6212b6b6 | 324 | if (likely((long) (commit_seq_old - commit_count) < 0)) |
8ec496cf | 325 | v_set(config, &cc_hot->seq, commit_count); |
f3bc08c5 MD |
326 | } |
327 | ||
e20c0fec | 328 | extern int lib_ring_buffer_create(struct lttng_kernel_ring_buffer *buf, |
f3bc08c5 | 329 | struct channel_backend *chanb, int cpu); |
e20c0fec | 330 | extern void lib_ring_buffer_free(struct lttng_kernel_ring_buffer *buf); |
f3bc08c5 MD |
331 | |
332 | /* Keep track of trap nesting inside ring buffer code */ | |
333 | DECLARE_PER_CPU(unsigned int, lib_ring_buffer_nesting); | |
334 | ||
886d51a3 | 335 | #endif /* _LIB_RING_BUFFER_FRONTEND_INTERNAL_H */ |