2 /* LTTng user-space "fast" tracing header
4 * Copyright 2006 Mathieu Desnoyers
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
14 #include <asm/atomic.h>
18 #include <linux/futex.h>
19 #include <asm/timex.h>
21 #include <ltt/ltt-facility-id-user_generic.h>
22 #include <ltt/ltt-generic.h>
25 static inline __attribute__((no_instrument_function
))
26 _syscall6(long, futex
, unsigned long, uaddr
, int, op
, int, val
,
27 unsigned long, timeout
, unsigned long, uaddr2
, int, val2
)
33 #define LTT_N_SUBBUFS 2
34 #endif //LTT_N_SUBBUFS
36 #ifndef LTT_SUBBUF_SIZE_PROCESS
37 #define LTT_SUBBUF_SIZE_PROCESS 1048576
38 #endif //LTT_BUF_SIZE_CPU
40 #define LTT_BUF_SIZE_PROCESS (LTT_SUBBUF_SIZE_PROCESS * LTT_N_SUBBUFS)
42 #ifndef LTT_USERTRACE_ROOT
43 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
44 #endif //LTT_USERTRACE_ROOT
47 /* Buffer offset macros */
49 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
50 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
51 #define SUBBUF_ALIGN(offset, buf) \
52 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
53 #define SUBBUF_TRUNC(offset, buf) \
54 ((offset) & (~(buf->subbuf_size-1)))
55 #define SUBBUF_INDEX(offset, buf) \
56 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
59 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
60 #define LTT_TRACER_VERSION_MAJOR 0
61 #define LTT_TRACER_VERSION_MINOR 7
63 #ifndef atomic_cmpxchg
64 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
65 #endif //atomic_cmpxchg
67 typedef unsigned int ltt_facility_t
;
69 struct ltt_trace_header
{
70 uint32_t magic_number
;
72 uint32_t arch_variant
;
73 uint32_t float_word_order
; /* Only useful for user space traces */
75 //uint32_t system_type;
76 uint8_t major_version
;
77 uint8_t minor_version
;
78 uint8_t flight_recorder
;
79 uint8_t has_heartbeat
;
80 uint8_t has_alignment
; /* Event header alignment */
84 uint64_t start_monotonic
;
85 uint64_t start_time_sec
;
86 uint64_t start_time_usec
;
87 } __attribute((packed
));
90 struct ltt_block_start_header
{
93 uint64_t freq
; /* khz */
97 uint64_t freq
; /* khz */
99 uint32_t lost_size
; /* Size unused at the end of the buffer */
100 uint32_t buf_size
; /* The size of this sub-buffer */
101 struct ltt_trace_header trace
;
102 } __attribute((packed
));
110 atomic_t reserve_count
[LTT_N_SUBBUFS
];
111 atomic_t commit_count
[LTT_N_SUBBUFS
];
113 atomic_t events_lost
;
114 atomic_t corrupted_subbuffers
;
115 atomic_t writer_futex
; /* futex on which the writer waits */
116 unsigned int alloc_size
;
117 unsigned int subbuf_size
;
120 struct ltt_trace_info
{
126 struct ltt_buf process
;
127 char process_buf
[LTT_BUF_SIZE_PROCESS
] __attribute__ ((aligned (8)));
132 struct ltt_event_header_nohb
{
134 unsigned char facility_id
;
135 unsigned char event_id
;
137 } __attribute((packed
));
139 extern __thread
struct ltt_trace_info
*thread_trace_info
;
141 void ltt_thread_init(void);
143 void __attribute__((no_instrument_function
))
144 ltt_usertrace_fast_buffer_switch(void);
146 /* Get the offset of the channel in the ltt_trace_struct */
147 #define GET_CHANNEL_INDEX(chan) \
148 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
150 /* ltt_get_index_from_facility
152 * Get channel index from facility and event id.
155 * @eID : event number
157 * Get the channel index into which events must be written for the given
158 * facility and event number. We get this structure offset as soon as possible
159 * and remember it so we pass through this logic only once per trace call (not
162 static inline unsigned int __attribute__((no_instrument_function
))
163 ltt_get_index_from_facility(ltt_facility_t fID
,
166 return GET_CHANNEL_INDEX(process
);
170 static inline struct ltt_buf
* __attribute__((no_instrument_function
))
171 ltt_get_channel_from_index(
172 struct ltt_trace_info
*trace
, unsigned int index
)
174 return (struct ltt_buf
*)((void*)trace
+index
);
179 * ltt_get_header_size
181 * Calculate alignment offset for arch size void*. This is the
182 * alignment offset of the event header.
185 * The event header must be a size multiple of the void* size. This is necessary
186 * to be able to calculate statically the alignment offset of the variable
187 * length data fields that follows. The total offset calculated here :
189 * Alignment of header struct on arch size
190 * + sizeof(header struct)
191 * + padding added to end of struct to align on arch size.
193 static inline unsigned char __attribute__((no_instrument_function
))
194 ltt_get_header_size(struct ltt_trace_info
*trace
,
196 size_t *before_hdr_pad
,
197 size_t *after_hdr_pad
,
200 unsigned int padding
;
203 header
= sizeof(struct ltt_event_header_nohb
);
205 /* Padding before the header. Calculated dynamically */
206 *before_hdr_pad
= ltt_align((unsigned long)address
, header
);
207 padding
= *before_hdr_pad
;
209 /* Padding after header, considering header aligned on ltt_align.
210 * Calculated statically if header size if known. */
211 *after_hdr_pad
= ltt_align(header
, sizeof(void*));
212 padding
+= *after_hdr_pad
;
214 *header_size
= header
;
216 return header
+padding
;
220 /* ltt_write_event_header
222 * Writes the event header to the pointer.
224 * @channel : pointer to the channel structure
225 * @ptr : buffer pointer
228 * @event_size : size of the event, excluding the event header.
229 * @offset : offset of the beginning of the header, for alignment.
230 * Calculated by ltt_get_event_header_size.
231 * @tsc : time stamp counter.
233 static inline void __attribute__((no_instrument_function
))
234 ltt_write_event_header(
235 struct ltt_trace_info
*trace
, struct ltt_buf
*buf
,
236 void *ptr
, ltt_facility_t fID
, uint32_t eID
, size_t event_size
,
237 size_t offset
, uint64_t tsc
)
239 struct ltt_event_header_nohb
*nohb
;
241 event_size
= min(event_size
, 0xFFFFU
);
242 nohb
= (struct ltt_event_header_nohb
*)(ptr
+offset
);
243 nohb
->timestamp
= (uint64_t)tsc
;
244 nohb
->facility_id
= fID
;
245 nohb
->event_id
= eID
;
246 nohb
->event_size
= (uint16_t)event_size
;
251 static inline uint64_t __attribute__((no_instrument_function
))
257 static inline unsigned int __attribute__((no_instrument_function
))
258 ltt_subbuf_header_len(struct ltt_buf
*buf
)
260 return sizeof(struct ltt_block_start_header
);
265 static inline void __attribute__((no_instrument_function
))
266 ltt_write_trace_header(struct ltt_trace_header
*header
)
268 header
->magic_number
= LTT_TRACER_MAGIC_NUMBER
;
269 header
->major_version
= LTT_TRACER_VERSION_MAJOR
;
270 header
->minor_version
= LTT_TRACER_VERSION_MINOR
;
271 header
->float_word_order
= 0; //FIXME
272 header
->arch_type
= 0; //FIXME LTT_ARCH_TYPE;
273 header
->arch_size
= sizeof(void*);
274 header
->arch_variant
= 0; //FIXME LTT_ARCH_VARIANT;
275 header
->flight_recorder
= 0;
276 header
->has_heartbeat
= 0;
279 header
->has_alignment
= sizeof(void*);
281 header
->has_alignment
= 0;
285 header
->freq_scale
= 0;
286 header
->start_freq
= 0;
287 header
->start_tsc
= 0;
288 header
->start_monotonic
= 0;
289 header
->start_time_sec
= 0;
290 header
->start_time_usec
= 0;
294 static inline void __attribute__((no_instrument_function
))
295 ltt_buffer_begin_callback(struct ltt_buf
*buf
,
296 uint64_t tsc
, unsigned int subbuf_idx
)
298 struct ltt_block_start_header
*header
=
299 (struct ltt_block_start_header
*)
300 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
302 header
->begin
.cycle_count
= tsc
;
303 header
->begin
.freq
= 0; //ltt_frequency();
305 header
->lost_size
= 0xFFFFFFFF; // for debugging...
307 header
->buf_size
= buf
->subbuf_size
;
309 ltt_write_trace_header(&header
->trace
);
315 static inline void __attribute__((no_instrument_function
))
316 ltt_buffer_end_callback(struct ltt_buf
*buf
,
317 uint64_t tsc
, unsigned int offset
, unsigned int subbuf_idx
)
319 struct ltt_block_start_header
*header
=
320 (struct ltt_block_start_header
*)
321 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
322 /* offset is assumed to never be 0 here : never deliver a completely
323 * empty subbuffer. */
324 /* The lost size is between 0 and subbuf_size-1 */
325 header
->lost_size
= SUBBUF_OFFSET((buf
->subbuf_size
- offset
),
327 header
->end
.cycle_count
= tsc
;
328 header
->end
.freq
= 0; //ltt_frequency();
332 static inline void __attribute__((no_instrument_function
))
333 ltt_deliver_callback(struct ltt_buf
*buf
,
337 ltt_usertrace_fast_buffer_switch();
343 * Atomic slot reservation in a LTTng buffer. It will take care of
344 * sub-buffer switching.
348 * @trace : the trace structure to log to.
349 * @buf : the buffer to reserve space into.
350 * @data_size : size of the variable length data to log.
351 * @slot_size : pointer to total size of the slot (out)
352 * @tsc : pointer to the tsc at the slot reservation (out)
353 * @before_hdr_pad : dynamic padding before the event header.
354 * @after_hdr_pad : dynamic padding after the event header.
356 * Return : NULL if not enough space, else returns the pointer
357 * to the beginning of the reserved slot. */
358 static inline void * __attribute__((no_instrument_function
)) ltt_reserve_slot(
359 struct ltt_trace_info
*trace
,
360 struct ltt_buf
*ltt_buf
,
361 unsigned int data_size
,
362 unsigned int *slot_size
,
364 size_t *before_hdr_pad
,
365 size_t *after_hdr_pad
,
368 int offset_begin
, offset_end
, offset_old
;
370 int begin_switch
, end_switch_current
, end_switch_old
;
371 int reserve_commit_diff
= 0;
373 int consumed_old
, consumed_new
;
374 int commit_count
, reserve_count
;
378 offset_old
= atomic_read(<t_buf
->offset
);
379 offset_begin
= offset_old
;
382 end_switch_current
= 0;
384 *tsc
= ltt_get_timestamp();
386 /* Error in getting the timestamp, event lost */
387 atomic_inc(<t_buf
->events_lost
);
391 if(SUBBUF_OFFSET(offset_begin
, ltt_buf
) == 0) {
392 begin_switch
= 1; /* For offset_begin */
394 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
395 before_hdr_pad
, after_hdr_pad
, header_size
)
398 if((SUBBUF_OFFSET(offset_begin
, ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
400 end_switch_old
= 1; /* For offset_old */
401 begin_switch
= 1; /* For offset_begin */
407 offset_begin
= SUBBUF_ALIGN(offset_begin
, ltt_buf
);
409 offset_begin
= offset_begin
+ ltt_subbuf_header_len(ltt_buf
);
410 /* Test new buffer integrity */
411 reserve_commit_diff
=
412 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
414 - atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
416 if(reserve_commit_diff
== 0) {
417 /* Next buffer not corrupted. */
418 //if((SUBBUF_TRUNC(offset_begin, ltt_buf)
419 // - SUBBUF_TRUNC(atomic_read(<t_buf->consumed), ltt_buf))
420 // >= ltt_buf->alloc_size) {
421 if(atomic_dec_return(<t_buf
->writer_futex
) >= 0) {
424 /* We block until the reader unblocks us */
425 atomic_set(<t_buf
->writer_futex
, -1);
426 /* We block until the reader tells us to wake up.
427 Signals will simply cause this loop to restart.
430 ret
= futex((unsigned long)<t_buf
->writer_futex
,
431 FUTEX_WAIT
, -1, 0, 0, 0);
432 } while(ret
!= 0 && ret
!= EWOULDBLOCK
);
434 /* go on with the write */
437 // /* next buffer not corrupted, we are either in overwrite mode or
438 // * the buffer is not full. It's safe to write in this new subbuffer.*/
441 /* Next subbuffer corrupted. Force pushing reader even in normal
442 * mode. It's safe to write in this new subbuffer. */
444 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
445 before_hdr_pad
, after_hdr_pad
, header_size
) + data_size
;
446 if((SUBBUF_OFFSET(offset_begin
,ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
447 /* Event too big for subbuffers, report error, don't complete
448 * the sub-buffer switch. */
449 atomic_inc(<t_buf
->events_lost
);
452 /* We just made a successful buffer switch and the event fits in the
453 * new subbuffer. Let's write. */
456 /* Event fits in the current buffer and we are not on a switch boundary.
457 * It's safe to write */
459 offset_end
= offset_begin
+ size
;
461 if((SUBBUF_OFFSET(offset_end
, ltt_buf
)) == 0) {
462 /* The offset_end will fall at the very beginning of the next subbuffer.
464 end_switch_current
= 1; /* For offset_begin */
467 } while(atomic_cmpxchg(<t_buf
->offset
, offset_old
, offset_end
)
471 /* Push the reader if necessary */
473 consumed_old
= atomic_read(<t_buf
->consumed
);
474 /* If buffer is in overwrite mode, push the reader consumed count if
475 the write position has reached it and we are not at the first
476 iteration (don't push the reader farther than the writer).
477 This operation can be done concurrently by many writers in the
478 same buffer, the writer being at the fartest write position sub-buffer
479 index in the buffer being the one which will win this loop. */
480 /* If the buffer is not in overwrite mode, pushing the reader only
481 happen if a sub-buffer is corrupted */
482 if((SUBBUF_TRUNC(offset_end
, ltt_buf
)
483 - SUBBUF_TRUNC(consumed_old
, ltt_buf
))
484 >= ltt_buf
->alloc_size
)
485 consumed_new
= SUBBUF_ALIGN(consumed_old
, ltt_buf
);
487 consumed_new
= consumed_old
;
490 } while(atomic_cmpxchg(<t_buf
->consumed
, consumed_old
, consumed_new
)
493 if(consumed_old
!= consumed_new
) {
494 /* Reader pushed : we are the winner of the push, we can therefore
495 reequilibrate reserve and commit. Atomic increment of the commit
496 count permits other writers to play around with this variable
497 before us. We keep track of corrupted_subbuffers even in overwrite mode :
498 we never want to write over a non completely committed sub-buffer :
499 possible causes : the buffer size is too low compared to the unordered
500 data input, or there is a writer who died between the reserve and the
502 if(reserve_commit_diff
) {
503 /* We have to alter the sub-buffer commit count : a sub-buffer is
504 corrupted. We do not deliver it. */
505 atomic_add(reserve_commit_diff
,
506 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
507 atomic_inc(<t_buf
->corrupted_subbuffers
);
514 /* Concurrency safe because we are the last and only thread to alter this
515 sub-buffer. As long as it is not delivered and read, no other thread can
516 alter the offset, alter the reserve_count or call the
517 client_buffer_end_callback on this sub-buffer.
518 The only remaining threads could be the ones with pending commits. They
519 will have to do the deliver themself.
520 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
521 with commit and reserve counts. We keep a corrupted sub-buffers count
522 and push the readers across these sub-buffers.
523 Not concurrency safe if a writer is stalled in a subbuffer and
524 another writer switches in, finding out it's corrupted. The result will
525 be than the old (uncommited) subbuffer will be declared corrupted, and
526 that the new subbuffer will be declared corrupted too because of the
527 commit count adjustment.
528 Note : offset_old should never be 0 here.*/
529 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_old
,
530 SUBBUF_INDEX((offset_old
-1), ltt_buf
));
531 /* Setting this reserve_count will allow the sub-buffer to be delivered by
532 the last committer. */
534 atomic_add_return((SUBBUF_OFFSET((offset_old
-1), ltt_buf
)+1),
535 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_old
-1), ltt_buf
)]);
537 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_old
-1),
539 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_old
-1), ltt_buf
),
546 /* This code can be executed unordered : writers may already have written
547 to the sub-buffer before this code gets executed, caution. */
548 /* The commit makes sure that this code is executed before the deliver
549 of this sub-buffer */
550 ltt_buffer_begin_callback(ltt_buf
, *tsc
, SUBBUF_INDEX(offset_begin
, ltt_buf
));
551 commit_count
= atomic_add_return(ltt_subbuf_header_len(ltt_buf
),
552 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
553 /* Check if the written buffer has to be delivered */
555 == atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
557 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
561 if(end_switch_current
) {
562 /* current subbuffer */
563 /* Concurrency safe because we are the last and only thread to alter this
564 sub-buffer. As long as it is not delivered and read, no other thread can
565 alter the offset, alter the reserve_count or call the
566 client_buffer_end_callback on this sub-buffer.
567 The only remaining threads could be the ones with pending commits. They
568 will have to do the deliver themself.
569 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
570 with commit and reserve counts. We keep a corrupted sub-buffers count
571 and push the readers across these sub-buffers.
572 Not concurrency safe if a writer is stalled in a subbuffer and
573 another writer switches in, finding out it's corrupted. The result will
574 be than the old (uncommited) subbuffer will be declared corrupted, and
575 that the new subbuffer will be declared corrupted too because of the
576 commit count adjustment. */
577 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_end
,
578 SUBBUF_INDEX((offset_end
-1), ltt_buf
));
579 /* Setting this reserve_count will allow the sub-buffer to be delivered by
580 the last committer. */
582 atomic_add_return((SUBBUF_OFFSET((offset_end
-1), ltt_buf
)+1),
583 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_end
-1), ltt_buf
)]);
585 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_end
-1),
587 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_end
-1), ltt_buf
), NULL
);
593 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
594 //BUG_ON(*slot_size != (offset_end - offset_begin));
596 return ltt_buf
->start
+ BUFFER_OFFSET(offset_begin
, ltt_buf
);
602 * Atomic unordered slot commit. Increments the commit count in the
603 * specified sub-buffer, and delivers it if necessary.
607 * @buf : the buffer to commit to.
608 * @reserved : address of the beginnig of the reserved slot.
609 * @slot_size : size of the reserved slot.
612 static inline void __attribute__((no_instrument_function
)) ltt_commit_slot(
613 struct ltt_buf
*ltt_buf
,
615 unsigned int slot_size
)
617 unsigned int offset_begin
= reserved
- ltt_buf
->start
;
620 commit_count
= atomic_add_return(slot_size
,
621 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
624 /* Check if all commits have been done */
626 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)])) {
627 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
635 #endif //_LTT_USERTRACE_FAST_H