Commit | Line | Data |
---|---|---|
b5b073e2 PMF |
1 | /* |
2 | * buffers.h | |
3 | * | |
4 | * Copyright (C) 2009 - Pierre-Marc Fournier (pierre-marc dot fournier at polymtl dot ca) | |
5 | * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca) | |
6 | * | |
7 | */ | |
8 | ||
9 | #ifndef _UST_BUFFERS_H | |
10 | #define _UST_BUFFERS_H | |
11 | ||
12 | #include <kcompat/kref.h> | |
13 | #include <assert.h> | |
14 | #include "channels.h" | |
15 | #include "buffers.h" | |
16 | ||
17 | /* Return the size of the minimum number of pages that can contain x. */ | |
18 | #define FIX_SIZE(x) ((((x) - 1) & PAGE_MASK) + PAGE_SIZE) | |
19 | ||
20 | /* | |
21 | * BUFFER_TRUNC zeroes the subbuffer offset and the subbuffer number parts of | |
22 | * the offset, which leaves only the buffer number. | |
23 | */ | |
24 | #define BUFFER_TRUNC(offset, chan) \ | |
25 | ((offset) & (~((chan)->alloc_size-1))) | |
26 | #define BUFFER_OFFSET(offset, chan) ((offset) & ((chan)->alloc_size - 1)) | |
27 | #define SUBBUF_OFFSET(offset, chan) ((offset) & ((chan)->subbuf_size - 1)) | |
28 | #define SUBBUF_ALIGN(offset, chan) \ | |
29 | (((offset) + (chan)->subbuf_size) & (~((chan)->subbuf_size - 1))) | |
30 | #define SUBBUF_TRUNC(offset, chan) \ | |
31 | ((offset) & (~((chan)->subbuf_size - 1))) | |
32 | #define SUBBUF_INDEX(offset, chan) \ | |
33 | (BUFFER_OFFSET((offset), chan) >> (chan)->subbuf_size_order) | |
34 | ||
35 | /* | |
36 | * Tracks changes to rchan/rchan_buf structs | |
37 | */ | |
38 | #define UST_CHANNEL_VERSION 8 | |
39 | ||
40 | struct ust_buffer { | |
41 | /* First 32 bytes cache-hot cacheline */ | |
42 | local_t offset; /* Current offset in the buffer */ | |
43 | local_t *commit_count; /* Commit count per sub-buffer */ | |
44 | atomic_long_t consumed; /* | |
45 | * Current offset in the buffer | |
46 | * standard atomic access (shared) | |
47 | */ | |
48 | unsigned long last_tsc; /* | |
49 | * Last timestamp written in the buffer. | |
50 | */ | |
51 | /* End of first 32 bytes cacheline */ | |
52 | atomic_long_t active_readers; /* | |
53 | * Active readers count | |
54 | * standard atomic access (shared) | |
55 | */ | |
56 | local_t events_lost; | |
57 | local_t corrupted_subbuffers; | |
58 | /* one byte is written to this pipe when data is available, in order | |
59 | to wake the consumer */ | |
60 | /* portability: Single byte writes must be as quick as possible. The kernel-side | |
61 | buffer must be large enough so the writer doesn't block. From the pipe(7) | |
62 | man page: Since linux 2.6.11, the pipe capacity is 65536 bytes. */ | |
63 | int data_ready_fd_write; | |
64 | /* the reading end of the pipe */ | |
65 | int data_ready_fd_read; | |
66 | ||
67 | struct ust_channel *chan; | |
68 | struct kref kref; | |
69 | void *buf_data; | |
70 | size_t buf_size; | |
71 | int shmid; | |
72 | ||
73 | /* commit count per subbuffer; must be at end of struct */ | |
74 | local_t commit_seq[0] ____cacheline_aligned; | |
75 | } ____cacheline_aligned; | |
76 | ||
77 | extern void _ust_buffers_write(struct ust_buffer *buf, size_t offset, | |
78 | const void *src, size_t len, ssize_t cpy); | |
79 | ||
80 | /* | |
81 | * Return the address where a given offset is located. | |
82 | * Should be used to get the current subbuffer header pointer. Given we know | |
83 | * it's never on a page boundary, it's safe to write directly to this address, | |
84 | * as long as the write is never bigger than a page size. | |
85 | */ | |
86 | extern void *ust_buffers_offset_address(struct ust_buffer *buf, | |
87 | size_t offset); | |
88 | ||
89 | /* FIXME: lttng has a version for systems with inefficient unaligned access */ | |
a2fd50ef | 90 | static __inline__ void ust_buffers_do_copy(void *dest, const void *src, size_t len) |
b5b073e2 PMF |
91 | { |
92 | union { | |
93 | const void *src; | |
94 | const u8 *src8; | |
95 | const u16 *src16; | |
96 | const u32 *src32; | |
97 | const u64 *src64; | |
98 | } u = { .src = src }; | |
99 | ||
100 | switch (len) { | |
101 | case 0: break; | |
102 | case 1: *(u8 *)dest = *u.src8; | |
103 | break; | |
104 | case 2: *(u16 *)dest = *u.src16; | |
105 | break; | |
106 | case 4: *(u32 *)dest = *u.src32; | |
107 | break; | |
108 | case 8: *(u64 *)dest = *u.src64; | |
109 | break; | |
110 | default: | |
111 | memcpy(dest, src, len); | |
112 | } | |
113 | } | |
114 | ||
115 | /* FIXME: there is both a static inline and a '_' non static inline version ?? */ | |
a2fd50ef | 116 | static __inline__ int ust_buffers_write(struct ust_buffer *buf, size_t offset, |
b5b073e2 PMF |
117 | const void *src, size_t len) |
118 | { | |
119 | size_t cpy; | |
120 | size_t buf_offset = BUFFER_OFFSET(offset, buf->chan); | |
121 | ||
122 | assert(buf_offset < buf->chan->subbuf_size*buf->chan->subbuf_cnt); | |
123 | ||
124 | cpy = min_t(size_t, len, buf->buf_size - buf_offset); | |
125 | ust_buffers_do_copy(buf->buf_data + buf_offset, src, cpy); | |
126 | ||
127 | if (unlikely(len != cpy)) | |
128 | _ust_buffers_write(buf, buf_offset, src, len, cpy); | |
129 | return len; | |
130 | } | |
131 | ||
132 | int ust_buffers_channel_open(struct ust_channel *chan, size_t subbuf_size, size_t n_subbufs); | |
133 | extern void ust_buffers_channel_close(struct ust_channel *chan); | |
134 | ||
135 | extern int ust_buffers_do_get_subbuf(struct ust_buffer *buf, long *pconsumed_old); | |
136 | ||
137 | extern int ust_buffers_do_put_subbuf(struct ust_buffer *buf, u32 uconsumed_old); | |
138 | ||
139 | extern void init_ustrelay_transport(void); | |
140 | ||
141 | /*static*/ /* inline */ notrace void ltt_commit_slot( | |
142 | struct ust_channel *channel, | |
143 | void **transport_data, long buf_offset, | |
144 | size_t data_size, size_t slot_size); | |
145 | ||
146 | #endif /* _UST_BUFFERS_H */ |