| 1 | /* |
| 2 | * buffers.h |
| 3 | * |
| 4 | * Copyright (C) 2009 - Pierre-Marc Fournier (pierre-marc dot fournier at polymtl dot ca) |
| 5 | * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca) |
| 6 | * |
| 7 | */ |
| 8 | |
| 9 | #ifndef _UST_BUFFERS_H |
| 10 | #define _UST_BUFFERS_H |
| 11 | |
| 12 | #include <kcompat/kref.h> |
| 13 | #include <assert.h> |
| 14 | #include "channels.h" |
| 15 | #include "buffers.h" |
| 16 | |
| 17 | /* Return the size of the minimum number of pages that can contain x. */ |
| 18 | #define FIX_SIZE(x) ((((x) - 1) & PAGE_MASK) + PAGE_SIZE) |
| 19 | |
| 20 | /* |
| 21 | * BUFFER_TRUNC zeroes the subbuffer offset and the subbuffer number parts of |
| 22 | * the offset, which leaves only the buffer number. |
| 23 | */ |
| 24 | #define BUFFER_TRUNC(offset, chan) \ |
| 25 | ((offset) & (~((chan)->alloc_size-1))) |
| 26 | #define BUFFER_OFFSET(offset, chan) ((offset) & ((chan)->alloc_size - 1)) |
| 27 | #define SUBBUF_OFFSET(offset, chan) ((offset) & ((chan)->subbuf_size - 1)) |
| 28 | #define SUBBUF_ALIGN(offset, chan) \ |
| 29 | (((offset) + (chan)->subbuf_size) & (~((chan)->subbuf_size - 1))) |
| 30 | #define SUBBUF_TRUNC(offset, chan) \ |
| 31 | ((offset) & (~((chan)->subbuf_size - 1))) |
| 32 | #define SUBBUF_INDEX(offset, chan) \ |
| 33 | (BUFFER_OFFSET((offset), chan) >> (chan)->subbuf_size_order) |
| 34 | |
| 35 | /* |
| 36 | * Tracks changes to rchan/rchan_buf structs |
| 37 | */ |
| 38 | #define UST_CHANNEL_VERSION 8 |
| 39 | |
| 40 | struct ust_buffer { |
| 41 | /* First 32 bytes cache-hot cacheline */ |
| 42 | local_t offset; /* Current offset in the buffer */ |
| 43 | local_t *commit_count; /* Commit count per sub-buffer */ |
| 44 | atomic_long_t consumed; /* |
| 45 | * Current offset in the buffer |
| 46 | * standard atomic access (shared) |
| 47 | */ |
| 48 | unsigned long last_tsc; /* |
| 49 | * Last timestamp written in the buffer. |
| 50 | */ |
| 51 | /* End of first 32 bytes cacheline */ |
| 52 | atomic_long_t active_readers; /* |
| 53 | * Active readers count |
| 54 | * standard atomic access (shared) |
| 55 | */ |
| 56 | local_t events_lost; |
| 57 | local_t corrupted_subbuffers; |
| 58 | /* one byte is written to this pipe when data is available, in order |
| 59 | to wake the consumer */ |
| 60 | /* portability: Single byte writes must be as quick as possible. The kernel-side |
| 61 | buffer must be large enough so the writer doesn't block. From the pipe(7) |
| 62 | man page: Since linux 2.6.11, the pipe capacity is 65536 bytes. */ |
| 63 | int data_ready_fd_write; |
| 64 | /* the reading end of the pipe */ |
| 65 | int data_ready_fd_read; |
| 66 | |
| 67 | struct ust_channel *chan; |
| 68 | struct kref kref; |
| 69 | void *buf_data; |
| 70 | size_t buf_size; |
| 71 | int shmid; |
| 72 | unsigned int cpu; |
| 73 | |
| 74 | /* commit count per subbuffer; must be at end of struct */ |
| 75 | local_t commit_seq[0] ____cacheline_aligned; |
| 76 | } ____cacheline_aligned; |
| 77 | |
| 78 | extern void _ust_buffers_write(struct ust_buffer *buf, size_t offset, |
| 79 | const void *src, size_t len, ssize_t cpy); |
| 80 | |
| 81 | /* |
| 82 | * Return the address where a given offset is located. |
| 83 | * Should be used to get the current subbuffer header pointer. Given we know |
| 84 | * it's never on a page boundary, it's safe to write directly to this address, |
| 85 | * as long as the write is never bigger than a page size. |
| 86 | */ |
| 87 | extern void *ust_buffers_offset_address(struct ust_buffer *buf, |
| 88 | size_t offset); |
| 89 | |
| 90 | /* FIXME: lttng has a version for systems with inefficient unaligned access */ |
| 91 | static __inline__ void ust_buffers_do_copy(void *dest, const void *src, size_t len) |
| 92 | { |
| 93 | union { |
| 94 | const void *src; |
| 95 | const u8 *src8; |
| 96 | const u16 *src16; |
| 97 | const u32 *src32; |
| 98 | const u64 *src64; |
| 99 | } u = { .src = src }; |
| 100 | |
| 101 | switch (len) { |
| 102 | case 0: break; |
| 103 | case 1: *(u8 *)dest = *u.src8; |
| 104 | break; |
| 105 | case 2: *(u16 *)dest = *u.src16; |
| 106 | break; |
| 107 | case 4: *(u32 *)dest = *u.src32; |
| 108 | break; |
| 109 | case 8: *(u64 *)dest = *u.src64; |
| 110 | break; |
| 111 | default: |
| 112 | memcpy(dest, src, len); |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | /* FIXME: there is both a static inline and a '_' non static inline version ?? */ |
| 117 | static __inline__ int ust_buffers_write(struct ust_buffer *buf, size_t offset, |
| 118 | const void *src, size_t len) |
| 119 | { |
| 120 | size_t cpy; |
| 121 | size_t buf_offset = BUFFER_OFFSET(offset, buf->chan); |
| 122 | |
| 123 | assert(buf_offset < buf->chan->subbuf_size*buf->chan->subbuf_cnt); |
| 124 | |
| 125 | cpy = min_t(size_t, len, buf->buf_size - buf_offset); |
| 126 | ust_buffers_do_copy(buf->buf_data + buf_offset, src, cpy); |
| 127 | |
| 128 | if (unlikely(len != cpy)) |
| 129 | _ust_buffers_write(buf, buf_offset, src, len, cpy); |
| 130 | return len; |
| 131 | } |
| 132 | |
| 133 | int ust_buffers_channel_open(struct ust_channel *chan, size_t subbuf_size, size_t n_subbufs); |
| 134 | extern void ust_buffers_channel_close(struct ust_channel *chan); |
| 135 | |
| 136 | extern int ust_buffers_do_get_subbuf(struct ust_buffer *buf, long *pconsumed_old); |
| 137 | |
| 138 | extern int ust_buffers_do_put_subbuf(struct ust_buffer *buf, u32 uconsumed_old); |
| 139 | |
| 140 | extern void init_ustrelay_transport(void); |
| 141 | |
| 142 | /*static*/ /* inline */ notrace void ltt_commit_slot( |
| 143 | struct ust_channel *channel, |
| 144 | void **transport_data, long buf_offset, |
| 145 | size_t data_size, size_t slot_size); |
| 146 | |
| 147 | #endif /* _UST_BUFFERS_H */ |