2 /* LTTng user-space "fast" tracing header
4 * Copyright 2006 Mathieu Desnoyers
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
14 #include <asm/atomic.h>
18 #include <linux/futex.h>
19 #include <asm/timex.h>
21 #include <ltt/ltt-facility-id-user_generic.h>
22 #include <ltt/ltt-generic.h>
25 static inline __attribute__((no_instrument_function
))
26 _syscall6(long, futex
, unsigned long, uaddr
, int, op
, int, val
,
27 unsigned long, timeout
, unsigned long, uaddr2
, int, val2
)
33 #define LTT_N_SUBBUFS 2
34 #endif //LTT_N_SUBBUFS
36 #ifndef LTT_SUBBUF_SIZE_CPU
37 #define LTT_SUBBUF_SIZE_CPU 1048576
38 #endif //LTT_BUF_SIZE_CPU
40 #define LTT_BUF_SIZE_CPU (LTT_SUBBUF_SIZE_CPU * LTT_N_SUBBUFS)
42 #ifndef LTT_SUBBUF_SIZE_FACILITIES
43 #define LTT_SUBBUF_SIZE_FACILITIES 4096
44 #endif //LTT_BUF_SIZE_FACILITIES
46 #define LTT_BUF_SIZE_FACILITIES (LTT_SUBBUF_SIZE_FACILITIES * LTT_N_SUBBUFS)
48 #ifndef LTT_USERTRACE_ROOT
49 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
50 #endif //LTT_USERTRACE_ROOT
53 /* Buffer offset macros */
55 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
56 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
57 #define SUBBUF_ALIGN(offset, buf) \
58 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
59 #define SUBBUF_TRUNC(offset, buf) \
60 ((offset) & (~(buf->subbuf_size-1)))
61 #define SUBBUF_INDEX(offset, buf) \
62 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
65 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
66 #define LTT_TRACER_VERSION_MAJOR 0
67 #define LTT_TRACER_VERSION_MINOR 7
69 #ifndef atomic_cmpxchg
70 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
71 #endif //atomic_cmpxchg
73 typedef unsigned int ltt_facility_t
;
75 struct ltt_trace_header
{
76 uint32_t magic_number
;
78 uint32_t arch_variant
;
79 uint32_t float_word_order
; /* Only useful for user space traces */
81 //uint32_t system_type;
82 uint8_t major_version
;
83 uint8_t minor_version
;
84 uint8_t flight_recorder
;
85 uint8_t has_heartbeat
;
86 uint8_t has_alignment
; /* Event header alignment */
90 uint64_t start_monotonic
;
91 uint64_t start_time_sec
;
92 uint64_t start_time_usec
;
93 } __attribute((packed
));
96 struct ltt_block_start_header
{
99 uint64_t freq
; /* khz */
102 uint64_t cycle_count
;
103 uint64_t freq
; /* khz */
105 uint32_t lost_size
; /* Size unused at the end of the buffer */
106 uint32_t buf_size
; /* The size of this sub-buffer */
107 struct ltt_trace_header trace
;
108 } __attribute((packed
));
116 atomic_t reserve_count
[LTT_N_SUBBUFS
];
117 atomic_t commit_count
[LTT_N_SUBBUFS
];
119 atomic_t events_lost
;
120 atomic_t corrupted_subbuffers
;
121 atomic_t full
; /* futex on which the writer waits : 1 : full */
122 unsigned int alloc_size
;
123 unsigned int subbuf_size
;
126 struct ltt_trace_info
{
132 struct ltt_buf facilities
;
134 char facilities_buf
[LTT_BUF_SIZE_FACILITIES
] __attribute__ ((aligned (8)));
135 char cpu_buf
[LTT_BUF_SIZE_CPU
] __attribute__ ((aligned (8)));
140 struct ltt_event_header_nohb
{
142 unsigned char facility_id
;
143 unsigned char event_id
;
145 } __attribute((packed
));
147 extern __thread
struct ltt_trace_info
*thread_trace_info
;
149 void ltt_thread_init(void);
151 void __attribute__((no_instrument_function
))
152 ltt_usertrace_fast_buffer_switch(void);
154 /* Get the offset of the channel in the ltt_trace_struct */
155 #define GET_CHANNEL_INDEX(chan) \
156 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
158 /* ltt_get_index_from_facility
160 * Get channel index from facility and event id.
163 * @eID : event number
165 * Get the channel index into which events must be written for the given
166 * facility and event number. We get this structure offset as soon as possible
167 * and remember it so we pass through this logic only once per trace call (not
170 static inline unsigned int __attribute__((no_instrument_function
))
171 ltt_get_index_from_facility(ltt_facility_t fID
,
174 return GET_CHANNEL_INDEX(cpu
);
178 static inline struct ltt_buf
* __attribute__((no_instrument_function
))
179 ltt_get_channel_from_index(
180 struct ltt_trace_info
*trace
, unsigned int index
)
182 return (struct ltt_buf
*)((void*)trace
+index
);
187 * ltt_get_header_size
189 * Calculate alignment offset for arch size void*. This is the
190 * alignment offset of the event header.
193 * The event header must be a size multiple of the void* size. This is necessary
194 * to be able to calculate statically the alignment offset of the variable
195 * length data fields that follows. The total offset calculated here :
197 * Alignment of header struct on arch size
198 * + sizeof(header struct)
199 * + padding added to end of struct to align on arch size.
201 static inline unsigned char __attribute__((no_instrument_function
))
202 ltt_get_header_size(struct ltt_trace_info
*trace
,
204 size_t *before_hdr_pad
,
205 size_t *after_hdr_pad
,
208 unsigned int padding
;
211 header
= sizeof(struct ltt_event_header_nohb
);
213 /* Padding before the header. Calculated dynamically */
214 *before_hdr_pad
= ltt_align((unsigned long)address
, header
);
215 padding
= *before_hdr_pad
;
217 /* Padding after header, considering header aligned on ltt_align.
218 * Calculated statically if header size if known. */
219 *after_hdr_pad
= ltt_align(header
, sizeof(void*));
220 padding
+= *after_hdr_pad
;
222 *header_size
= header
;
224 return header
+padding
;
228 /* ltt_write_event_header
230 * Writes the event header to the pointer.
232 * @channel : pointer to the channel structure
233 * @ptr : buffer pointer
236 * @event_size : size of the event, excluding the event header.
237 * @offset : offset of the beginning of the header, for alignment.
238 * Calculated by ltt_get_event_header_size.
239 * @tsc : time stamp counter.
241 static inline void __attribute__((no_instrument_function
))
242 ltt_write_event_header(
243 struct ltt_trace_info
*trace
, struct ltt_buf
*buf
,
244 void *ptr
, ltt_facility_t fID
, uint32_t eID
, size_t event_size
,
245 size_t offset
, uint64_t tsc
)
247 struct ltt_event_header_nohb
*nohb
;
249 event_size
= min(event_size
, 0xFFFFU
);
250 nohb
= (struct ltt_event_header_nohb
*)(ptr
+offset
);
251 nohb
->timestamp
= (uint64_t)tsc
;
252 nohb
->facility_id
= fID
;
253 nohb
->event_id
= eID
;
254 nohb
->event_size
= (uint16_t)event_size
;
259 static inline uint64_t __attribute__((no_instrument_function
))
265 static inline unsigned int __attribute__((no_instrument_function
))
266 ltt_subbuf_header_len(struct ltt_buf
*buf
)
268 return sizeof(struct ltt_block_start_header
);
273 static inline void __attribute__((no_instrument_function
))
274 ltt_write_trace_header(struct ltt_trace_header
*header
)
276 header
->magic_number
= LTT_TRACER_MAGIC_NUMBER
;
277 header
->major_version
= LTT_TRACER_VERSION_MAJOR
;
278 header
->minor_version
= LTT_TRACER_VERSION_MINOR
;
279 header
->float_word_order
= 0; //FIXME
280 header
->arch_type
= 0; //FIXME LTT_ARCH_TYPE;
281 header
->arch_size
= sizeof(void*);
282 header
->arch_variant
= 0; //FIXME LTT_ARCH_VARIANT;
283 header
->flight_recorder
= 0;
284 header
->has_heartbeat
= 0;
287 header
->has_alignment
= sizeof(void*);
289 header
->has_alignment
= 0;
293 header
->freq_scale
= 0;
294 header
->start_freq
= 0;
295 header
->start_tsc
= 0;
296 header
->start_monotonic
= 0;
297 header
->start_time_sec
= 0;
298 header
->start_time_usec
= 0;
302 static inline void __attribute__((no_instrument_function
))
303 ltt_buffer_begin_callback(struct ltt_buf
*buf
,
304 uint64_t tsc
, unsigned int subbuf_idx
)
306 struct ltt_block_start_header
*header
=
307 (struct ltt_block_start_header
*)
308 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
310 header
->begin
.cycle_count
= tsc
;
311 header
->begin
.freq
= 0; //ltt_frequency();
313 header
->lost_size
= 0xFFFFFFFF; // for debugging...
315 header
->buf_size
= buf
->subbuf_size
;
317 ltt_write_trace_header(&header
->trace
);
323 static inline void __attribute__((no_instrument_function
))
324 ltt_buffer_end_callback(struct ltt_buf
*buf
,
325 uint64_t tsc
, unsigned int offset
, unsigned int subbuf_idx
)
327 struct ltt_block_start_header
*header
=
328 (struct ltt_block_start_header
*)
329 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
330 /* offset is assumed to never be 0 here : never deliver a completely
331 * empty subbuffer. */
332 /* The lost size is between 0 and subbuf_size-1 */
333 header
->lost_size
= SUBBUF_OFFSET((buf
->subbuf_size
- offset
),
335 header
->end
.cycle_count
= tsc
;
336 header
->end
.freq
= 0; //ltt_frequency();
340 static inline void __attribute__((no_instrument_function
))
341 ltt_deliver_callback(struct ltt_buf
*buf
,
345 ltt_usertrace_fast_buffer_switch();
351 * Atomic slot reservation in a LTTng buffer. It will take care of
352 * sub-buffer switching.
356 * @trace : the trace structure to log to.
357 * @buf : the buffer to reserve space into.
358 * @data_size : size of the variable length data to log.
359 * @slot_size : pointer to total size of the slot (out)
360 * @tsc : pointer to the tsc at the slot reservation (out)
361 * @before_hdr_pad : dynamic padding before the event header.
362 * @after_hdr_pad : dynamic padding after the event header.
364 * Return : NULL if not enough space, else returns the pointer
365 * to the beginning of the reserved slot. */
366 static inline void * __attribute__((no_instrument_function
)) ltt_reserve_slot(
367 struct ltt_trace_info
*trace
,
368 struct ltt_buf
*ltt_buf
,
369 unsigned int data_size
,
370 unsigned int *slot_size
,
372 size_t *before_hdr_pad
,
373 size_t *after_hdr_pad
,
376 int offset_begin
, offset_end
, offset_old
;
378 int begin_switch
, end_switch_current
, end_switch_old
;
379 int reserve_commit_diff
= 0;
381 int consumed_old
, consumed_new
;
382 int commit_count
, reserve_count
;
386 offset_old
= atomic_read(<t_buf
->offset
);
387 offset_begin
= offset_old
;
390 end_switch_current
= 0;
392 *tsc
= ltt_get_timestamp();
394 /* Error in getting the timestamp, event lost */
395 atomic_inc(<t_buf
->events_lost
);
399 if(SUBBUF_OFFSET(offset_begin
, ltt_buf
) == 0) {
400 begin_switch
= 1; /* For offset_begin */
402 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
403 before_hdr_pad
, after_hdr_pad
, header_size
)
406 if((SUBBUF_OFFSET(offset_begin
, ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
408 end_switch_old
= 1; /* For offset_old */
409 begin_switch
= 1; /* For offset_begin */
415 offset_begin
= SUBBUF_ALIGN(offset_begin
, ltt_buf
);
417 offset_begin
= offset_begin
+ ltt_subbuf_header_len(ltt_buf
);
418 /* Test new buffer integrity */
419 reserve_commit_diff
=
420 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
422 - atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
424 if(reserve_commit_diff
== 0) {
425 /* Next buffer not corrupted. */
426 if((SUBBUF_TRUNC(offset_begin
, ltt_buf
)
427 - SUBBUF_TRUNC(atomic_read(<t_buf
->consumed
), ltt_buf
))
428 >= ltt_buf
->alloc_size
) {
429 /* We block until the reader unblocks us */
430 atomic_set(<t_buf
->full
, 1);
431 /* We block until the reader tells us to wake up.
432 Signals will simply cause this loop to restart.
435 ret
= futex((unsigned long)<t_buf
->full
, FUTEX_WAIT
, 1, 0, 0, 0);
436 } while(ret
!= 0 && ret
!= EWOULDBLOCK
);
437 /* go on with the write */
440 /* next buffer not corrupted, we are either in overwrite mode or
441 * the buffer is not full. It's safe to write in this new subbuffer.*/
444 /* Next subbuffer corrupted. Force pushing reader even in normal
445 * mode. It's safe to write in this new subbuffer. */
447 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
448 before_hdr_pad
, after_hdr_pad
, header_size
) + data_size
;
449 if((SUBBUF_OFFSET(offset_begin
,ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
450 /* Event too big for subbuffers, report error, don't complete
451 * the sub-buffer switch. */
452 atomic_inc(<t_buf
->events_lost
);
455 /* We just made a successful buffer switch and the event fits in the
456 * new subbuffer. Let's write. */
459 /* Event fits in the current buffer and we are not on a switch boundary.
460 * It's safe to write */
462 offset_end
= offset_begin
+ size
;
464 if((SUBBUF_OFFSET(offset_end
, ltt_buf
)) == 0) {
465 /* The offset_end will fall at the very beginning of the next subbuffer.
467 end_switch_current
= 1; /* For offset_begin */
470 } while(atomic_cmpxchg(<t_buf
->offset
, offset_old
, offset_end
)
474 /* Push the reader if necessary */
476 consumed_old
= atomic_read(<t_buf
->consumed
);
477 /* If buffer is in overwrite mode, push the reader consumed count if
478 the write position has reached it and we are not at the first
479 iteration (don't push the reader farther than the writer).
480 This operation can be done concurrently by many writers in the
481 same buffer, the writer being at the fartest write position sub-buffer
482 index in the buffer being the one which will win this loop. */
483 /* If the buffer is not in overwrite mode, pushing the reader only
484 happen if a sub-buffer is corrupted */
485 if((SUBBUF_TRUNC(offset_end
, ltt_buf
)
486 - SUBBUF_TRUNC(consumed_old
, ltt_buf
))
487 >= ltt_buf
->alloc_size
)
488 consumed_new
= SUBBUF_ALIGN(consumed_old
, ltt_buf
);
490 consumed_new
= consumed_old
;
493 } while(atomic_cmpxchg(<t_buf
->consumed
, consumed_old
, consumed_new
)
496 if(consumed_old
!= consumed_new
) {
497 /* Reader pushed : we are the winner of the push, we can therefore
498 reequilibrate reserve and commit. Atomic increment of the commit
499 count permits other writers to play around with this variable
500 before us. We keep track of corrupted_subbuffers even in overwrite mode :
501 we never want to write over a non completely committed sub-buffer :
502 possible causes : the buffer size is too low compared to the unordered
503 data input, or there is a writer who died between the reserve and the
505 if(reserve_commit_diff
) {
506 /* We have to alter the sub-buffer commit count : a sub-buffer is
507 corrupted. We do not deliver it. */
508 atomic_add(reserve_commit_diff
,
509 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
510 atomic_inc(<t_buf
->corrupted_subbuffers
);
517 /* Concurrency safe because we are the last and only thread to alter this
518 sub-buffer. As long as it is not delivered and read, no other thread can
519 alter the offset, alter the reserve_count or call the
520 client_buffer_end_callback on this sub-buffer.
521 The only remaining threads could be the ones with pending commits. They
522 will have to do the deliver themself.
523 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
524 with commit and reserve counts. We keep a corrupted sub-buffers count
525 and push the readers across these sub-buffers.
526 Not concurrency safe if a writer is stalled in a subbuffer and
527 another writer switches in, finding out it's corrupted. The result will
528 be than the old (uncommited) subbuffer will be declared corrupted, and
529 that the new subbuffer will be declared corrupted too because of the
530 commit count adjustment.
531 Note : offset_old should never be 0 here.*/
532 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_old
,
533 SUBBUF_INDEX((offset_old
-1), ltt_buf
));
534 /* Setting this reserve_count will allow the sub-buffer to be delivered by
535 the last committer. */
537 atomic_add_return((SUBBUF_OFFSET((offset_old
-1), ltt_buf
)+1),
538 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_old
-1), ltt_buf
)]);
540 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_old
-1),
542 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_old
-1), ltt_buf
),
549 /* This code can be executed unordered : writers may already have written
550 to the sub-buffer before this code gets executed, caution. */
551 /* The commit makes sure that this code is executed before the deliver
552 of this sub-buffer */
553 ltt_buffer_begin_callback(ltt_buf
, *tsc
, SUBBUF_INDEX(offset_begin
, ltt_buf
));
554 commit_count
= atomic_add_return(ltt_subbuf_header_len(ltt_buf
),
555 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
556 /* Check if the written buffer has to be delivered */
558 == atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
560 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
564 if(end_switch_current
) {
565 /* current subbuffer */
566 /* Concurrency safe because we are the last and only thread to alter this
567 sub-buffer. As long as it is not delivered and read, no other thread can
568 alter the offset, alter the reserve_count or call the
569 client_buffer_end_callback on this sub-buffer.
570 The only remaining threads could be the ones with pending commits. They
571 will have to do the deliver themself.
572 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
573 with commit and reserve counts. We keep a corrupted sub-buffers count
574 and push the readers across these sub-buffers.
575 Not concurrency safe if a writer is stalled in a subbuffer and
576 another writer switches in, finding out it's corrupted. The result will
577 be than the old (uncommited) subbuffer will be declared corrupted, and
578 that the new subbuffer will be declared corrupted too because of the
579 commit count adjustment. */
580 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_end
,
581 SUBBUF_INDEX((offset_end
-1), ltt_buf
));
582 /* Setting this reserve_count will allow the sub-buffer to be delivered by
583 the last committer. */
585 atomic_add_return((SUBBUF_OFFSET((offset_end
-1), ltt_buf
)+1),
586 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_end
-1), ltt_buf
)]);
588 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_end
-1),
590 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_end
-1), ltt_buf
), NULL
);
596 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
597 //BUG_ON(*slot_size != (offset_end - offset_begin));
599 return ltt_buf
->start
+ BUFFER_OFFSET(offset_begin
, ltt_buf
);
605 * Atomic unordered slot commit. Increments the commit count in the
606 * specified sub-buffer, and delivers it if necessary.
610 * @buf : the buffer to commit to.
611 * @reserved : address of the beginnig of the reserved slot.
612 * @slot_size : size of the reserved slot.
615 static inline void __attribute__((no_instrument_function
)) ltt_commit_slot(
616 struct ltt_buf
*ltt_buf
,
618 unsigned int slot_size
)
620 unsigned int offset_begin
= reserved
- ltt_buf
->start
;
623 commit_count
= atomic_add_return(slot_size
,
624 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
627 /* Check if all commits have been done */
629 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)])) {
630 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
638 #endif //_LTT_USERTRACE_FAST_H