2 /* LTTng user-space "fast" tracing header
4 * Copyright 2006 Mathieu Desnoyers
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
18 #include <semaphore.h>
21 #include <ltt/ltt-facility-id-user_generic.h>
28 #define LTT_N_SUBBUFS 2
29 #endif //LTT_N_SUBBUFS
31 #ifndef LTT_SUBBUF_SIZE_PROCESS
32 #define LTT_SUBBUF_SIZE_PROCESS 1048576
33 #endif //LTT_BUF_SIZE_CPU
35 #define LTT_BUF_SIZE_PROCESS (LTT_SUBBUF_SIZE_PROCESS * LTT_N_SUBBUFS)
37 #ifndef LTT_USERTRACE_ROOT
38 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
39 #endif //LTT_USERTRACE_ROOT
42 /* Buffer offset macros */
44 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
45 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
46 #define SUBBUF_ALIGN(offset, buf) \
47 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
48 #define SUBBUF_TRUNC(offset, buf) \
49 ((offset) & (~(buf->subbuf_size-1)))
50 #define SUBBUF_INDEX(offset, buf) \
51 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
54 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
55 #define LTT_TRACER_VERSION_MAJOR 0
56 #define LTT_TRACER_VERSION_MINOR 8
58 #ifndef atomic_cmpxchg
59 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
60 #endif //atomic_cmpxchg
62 struct ltt_trace_header
{
63 uint32_t magic_number
;
65 uint32_t arch_variant
;
66 uint32_t float_word_order
; /* Only useful for user space traces */
68 //uint32_t system_type;
69 uint8_t major_version
;
70 uint8_t minor_version
;
71 uint8_t flight_recorder
;
72 uint8_t has_heartbeat
;
73 uint8_t has_alignment
; /* Event header alignment */
74 uint8_t tsc_lsb_truncate
;
79 uint64_t start_monotonic
;
80 uint64_t start_time_sec
;
81 uint64_t start_time_usec
;
82 } __attribute((packed
));
85 struct ltt_block_start_header
{
88 uint64_t freq
; /* khz */
92 uint64_t freq
; /* khz */
94 uint32_t lost_size
; /* Size unused at the end of the buffer */
95 uint32_t buf_size
; /* The size of this sub-buffer */
96 struct ltt_trace_header trace
;
97 } __attribute((packed
));
105 atomic_t reserve_count
[LTT_N_SUBBUFS
];
106 atomic_t commit_count
[LTT_N_SUBBUFS
];
108 atomic_t events_lost
;
109 atomic_t corrupted_subbuffers
;
110 sem_t writer_sem
; /* semaphore on which the writer waits */
111 unsigned int alloc_size
;
112 unsigned int subbuf_size
;
115 struct ltt_trace_info
{
121 struct ltt_buf process
;
122 char process_buf
[LTT_BUF_SIZE_PROCESS
] __attribute__ ((aligned (8)));
127 struct ltt_event_header_nohb
{
129 unsigned char facility_id
;
130 unsigned char event_id
;
132 } __attribute((packed
));
134 extern __thread
struct ltt_trace_info
*thread_trace_info
;
136 void ltt_thread_init(void);
138 void __attribute__((no_instrument_function
))
139 ltt_usertrace_fast_buffer_switch(void);
141 /* Get the offset of the channel in the ltt_trace_struct */
142 #define GET_CHANNEL_INDEX(chan) \
143 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
145 /* ltt_get_index_from_facility
147 * Get channel index from facility and event id.
150 * @eID : event number
152 * Get the channel index into which events must be written for the given
153 * facility and event number. We get this structure offset as soon as possible
154 * and remember it so we pass through this logic only once per trace call (not
157 static inline unsigned int __attribute__((no_instrument_function
))
158 ltt_get_index_from_facility(ltt_facility_t fID
,
161 return GET_CHANNEL_INDEX(process
);
165 static inline struct ltt_buf
* __attribute__((no_instrument_function
))
166 ltt_get_channel_from_index(
167 struct ltt_trace_info
*trace
, unsigned int index
)
169 return (struct ltt_buf
*)((void*)trace
+index
);
174 * ltt_get_header_size
176 * Calculate alignment offset for arch size void*. This is the
177 * alignment offset of the event header.
180 * The event header must be a size multiple of the void* size. This is necessary
181 * to be able to calculate statically the alignment offset of the variable
182 * length data fields that follows. The total offset calculated here :
184 * Alignment of header struct on arch size
185 * + sizeof(header struct)
186 * + padding added to end of struct to align on arch size.
188 static inline unsigned char __attribute__((no_instrument_function
))
189 ltt_get_header_size(struct ltt_trace_info
*trace
,
191 size_t *before_hdr_pad
,
192 size_t *after_hdr_pad
,
195 unsigned int padding
;
198 header
= sizeof(struct ltt_event_header_nohb
);
200 /* Padding before the header. Calculated dynamically */
201 *before_hdr_pad
= ltt_align((unsigned long)address
, header
);
202 padding
= *before_hdr_pad
;
204 /* Padding after header, considering header aligned on ltt_align.
205 * Calculated statically if header size if known. */
206 *after_hdr_pad
= ltt_align(header
, sizeof(void*));
207 padding
+= *after_hdr_pad
;
209 *header_size
= header
;
211 return header
+padding
;
215 /* ltt_write_event_header
217 * Writes the event header to the pointer.
219 * @channel : pointer to the channel structure
220 * @ptr : buffer pointer
223 * @event_size : size of the event, excluding the event header.
224 * @offset : offset of the beginning of the header, for alignment.
225 * Calculated by ltt_get_event_header_size.
226 * @tsc : time stamp counter.
228 static inline void __attribute__((no_instrument_function
))
229 ltt_write_event_header(
230 struct ltt_trace_info
*trace
, struct ltt_buf
*buf
,
231 void *ptr
, ltt_facility_t fID
, uint32_t eID
, size_t event_size
,
232 size_t offset
, uint64_t tsc
)
234 struct ltt_event_header_nohb
*nohb
;
236 event_size
= min(event_size
, 0xFFFFU
);
237 nohb
= (struct ltt_event_header_nohb
*)(ptr
+offset
);
238 nohb
->timestamp
= (uint64_t)tsc
;
239 nohb
->facility_id
= fID
;
240 nohb
->event_id
= eID
;
241 nohb
->event_size
= (uint16_t)event_size
;
246 static inline uint64_t __attribute__((no_instrument_function
))
252 static inline unsigned int __attribute__((no_instrument_function
))
253 ltt_subbuf_header_len(struct ltt_buf
*buf
)
255 return sizeof(struct ltt_block_start_header
);
260 static inline void __attribute__((no_instrument_function
))
261 ltt_write_trace_header(struct ltt_trace_header
*header
)
263 header
->magic_number
= LTT_TRACER_MAGIC_NUMBER
;
264 header
->major_version
= LTT_TRACER_VERSION_MAJOR
;
265 header
->minor_version
= LTT_TRACER_VERSION_MINOR
;
266 header
->float_word_order
= 0; //FIXME
267 header
->arch_type
= 0; //FIXME LTT_ARCH_TYPE;
268 header
->arch_size
= sizeof(void*);
269 header
->arch_variant
= 0; //FIXME LTT_ARCH_VARIANT;
270 header
->flight_recorder
= 0;
271 header
->has_heartbeat
= 0;
272 header
->tsc_lsb_truncate
= 0;
276 header
->has_alignment
= sizeof(void*);
278 header
->has_alignment
= 0;
282 header
->freq_scale
= 0;
283 header
->start_freq
= 0;
284 header
->start_tsc
= 0;
285 header
->start_monotonic
= 0;
286 header
->start_time_sec
= 0;
287 header
->start_time_usec
= 0;
291 static inline void __attribute__((no_instrument_function
))
292 ltt_buffer_begin_callback(struct ltt_buf
*buf
,
293 uint64_t tsc
, unsigned int subbuf_idx
)
295 struct ltt_block_start_header
*header
=
296 (struct ltt_block_start_header
*)
297 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
299 header
->begin
.cycle_count
= tsc
;
300 header
->begin
.freq
= 0; //ltt_frequency();
302 header
->lost_size
= 0xFFFFFFFF; // for debugging...
304 header
->buf_size
= buf
->subbuf_size
;
306 ltt_write_trace_header(&header
->trace
);
312 static inline void __attribute__((no_instrument_function
))
313 ltt_buffer_end_callback(struct ltt_buf
*buf
,
314 uint64_t tsc
, unsigned int offset
, unsigned int subbuf_idx
)
316 struct ltt_block_start_header
*header
=
317 (struct ltt_block_start_header
*)
318 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
319 /* offset is assumed to never be 0 here : never deliver a completely
320 * empty subbuffer. */
321 /* The lost size is between 0 and subbuf_size-1 */
322 header
->lost_size
= SUBBUF_OFFSET((buf
->subbuf_size
- offset
),
324 header
->end
.cycle_count
= tsc
;
325 header
->end
.freq
= 0; //ltt_frequency();
329 static inline void __attribute__((no_instrument_function
))
330 ltt_deliver_callback(struct ltt_buf
*buf
,
334 ltt_usertrace_fast_buffer_switch();
340 * Atomic slot reservation in a LTTng buffer. It will take care of
341 * sub-buffer switching.
345 * @trace : the trace structure to log to.
346 * @buf : the buffer to reserve space into.
347 * @data_size : size of the variable length data to log.
348 * @slot_size : pointer to total size of the slot (out)
349 * @tsc : pointer to the tsc at the slot reservation (out)
350 * @before_hdr_pad : dynamic padding before the event header.
351 * @after_hdr_pad : dynamic padding after the event header.
353 * Return : NULL if not enough space, else returns the pointer
354 * to the beginning of the reserved slot. */
355 static inline void * __attribute__((no_instrument_function
)) ltt_reserve_slot(
356 struct ltt_trace_info
*trace
,
357 struct ltt_buf
*ltt_buf
,
358 unsigned int data_size
,
361 size_t *before_hdr_pad
,
362 size_t *after_hdr_pad
,
365 int offset_begin
, offset_end
, offset_old
;
367 int begin_switch
, end_switch_current
, end_switch_old
;
368 int reserve_commit_diff
= 0;
370 int consumed_old
, consumed_new
;
371 int commit_count
, reserve_count
;
373 sigset_t oldset
, set
;
376 offset_old
= atomic_read(<t_buf
->offset
);
377 offset_begin
= offset_old
;
380 end_switch_current
= 0;
382 *tsc
= ltt_get_timestamp();
384 /* Error in getting the timestamp, event lost */
385 atomic_inc(<t_buf
->events_lost
);
389 if(SUBBUF_OFFSET(offset_begin
, ltt_buf
) == 0) {
390 begin_switch
= 1; /* For offset_begin */
392 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
393 before_hdr_pad
, after_hdr_pad
, header_size
)
396 if((SUBBUF_OFFSET(offset_begin
, ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
398 end_switch_old
= 1; /* For offset_old */
399 begin_switch
= 1; /* For offset_begin */
405 offset_begin
= SUBBUF_ALIGN(offset_begin
, ltt_buf
);
407 offset_begin
= offset_begin
+ ltt_subbuf_header_len(ltt_buf
);
408 /* Test new buffer integrity */
409 reserve_commit_diff
=
410 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
412 - atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
415 if(reserve_commit_diff
== 0) {
416 /* Next buffer not corrupted. */
417 //if((SUBBUF_TRUNC(offset_begin, ltt_buf)
418 // - SUBBUF_TRUNC(atomic_read(<t_buf->consumed), ltt_buf))
419 // >= ltt_buf->alloc_size) {
421 /* sem_wait is not signal safe. Disable signals around it.
422 * Signals are kept disabled to make sure we win the cmpxchg. */
423 /* Disable signals */
424 ret
= sigfillset(&set
);
425 if(ret
) perror("LTT Error in sigfillset\n");
427 ret
= pthread_sigmask(SIG_BLOCK
, &set
, &oldset
);
428 if(ret
) perror("LTT Error in pthread_sigmask\n");
430 /* We detect if a signal came between
431 * the offset read and signal disabling:
432 * if it is the case, then we restart
433 * the loop after reenabling signals. It
434 * means that it's a signal that has
435 * won the buffer switch.*/
436 if(offset_old
!= atomic_read(<t_buf
->offset
)) {
437 ret
= pthread_sigmask(SIG_SETMASK
, &oldset
, NULL
);
438 if(ret
) perror("LTT Error in pthread_sigmask\n");
441 /* If the offset is still the same, then
442 * we can safely proceed to do the
443 * buffer switch without being
444 * interrupted by a signal. */
445 sem_wait(<t_buf
->writer_sem
);
448 /* go on with the write */
451 // /* next buffer not corrupted, we are either in overwrite mode or
452 // * the buffer is not full. It's safe to write in this new subbuffer.*/
455 /* Next subbuffer corrupted. Force pushing reader even in normal
456 * mode. It's safe to write in this new subbuffer. */
457 /* No sem_post is required because we fall through without doing a
460 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
461 before_hdr_pad
, after_hdr_pad
, header_size
) + data_size
;
462 if((SUBBUF_OFFSET(offset_begin
,ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
463 /* Event too big for subbuffers, report error, don't complete
464 * the sub-buffer switch. */
465 atomic_inc(<t_buf
->events_lost
);
466 if(reserve_commit_diff
== 0) {
467 ret
= pthread_sigmask(SIG_SETMASK
, &oldset
, NULL
);
468 if(ret
) perror("LTT Error in pthread_sigmask\n");
472 /* We just made a successful buffer switch and the event fits in the
473 * new subbuffer. Let's write. */
476 /* Event fits in the current buffer and we are not on a switch boundary.
477 * It's safe to write */
479 offset_end
= offset_begin
+ size
;
481 if((SUBBUF_OFFSET(offset_end
, ltt_buf
)) == 0) {
482 /* The offset_end will fall at the very beginning of the next subbuffer.
484 end_switch_current
= 1; /* For offset_begin */
487 } while(atomic_cmpxchg(<t_buf
->offset
, offset_old
, offset_end
)
490 /* Push the reader if necessary */
492 consumed_old
= atomic_read(<t_buf
->consumed
);
493 /* If buffer is in overwrite mode, push the reader consumed count if
494 the write position has reached it and we are not at the first
495 iteration (don't push the reader farther than the writer).
496 This operation can be done concurrently by many writers in the
497 same buffer, the writer being at the fartest write position sub-buffer
498 index in the buffer being the one which will win this loop. */
499 /* If the buffer is not in overwrite mode, pushing the reader only
500 happen if a sub-buffer is corrupted */
501 if((SUBBUF_TRUNC(offset_end
-1, ltt_buf
)
502 - SUBBUF_TRUNC(consumed_old
, ltt_buf
))
503 >= ltt_buf
->alloc_size
)
504 consumed_new
= SUBBUF_ALIGN(consumed_old
, ltt_buf
);
506 consumed_new
= consumed_old
;
509 } while(atomic_cmpxchg(<t_buf
->consumed
, consumed_old
, consumed_new
)
512 if(consumed_old
!= consumed_new
) {
513 /* Reader pushed : we are the winner of the push, we can therefore
514 reequilibrate reserve and commit. Atomic increment of the commit
515 count permits other writers to play around with this variable
516 before us. We keep track of corrupted_subbuffers even in overwrite mode :
517 we never want to write over a non completely committed sub-buffer :
518 possible causes : the buffer size is too low compared to the unordered
519 data input, or there is a writer who died between the reserve and the
521 if(reserve_commit_diff
) {
522 /* We have to alter the sub-buffer commit count : a sub-buffer is
523 corrupted. We do not deliver it. */
524 atomic_add(reserve_commit_diff
,
525 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
526 atomic_inc(<t_buf
->corrupted_subbuffers
);
533 /* Concurrency safe because we are the last and only thread to alter this
534 sub-buffer. As long as it is not delivered and read, no other thread can
535 alter the offset, alter the reserve_count or call the
536 client_buffer_end_callback on this sub-buffer.
537 The only remaining threads could be the ones with pending commits. They
538 will have to do the deliver themself.
539 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
540 with commit and reserve counts. We keep a corrupted sub-buffers count
541 and push the readers across these sub-buffers.
542 Not concurrency safe if a writer is stalled in a subbuffer and
543 another writer switches in, finding out it's corrupted. The result will
544 be than the old (uncommited) subbuffer will be declared corrupted, and
545 that the new subbuffer will be declared corrupted too because of the
546 commit count adjustment.
547 Note : offset_old should never be 0 here.*/
548 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_old
,
549 SUBBUF_INDEX((offset_old
-1), ltt_buf
));
550 /* Setting this reserve_count will allow the sub-buffer to be delivered by
551 the last committer. */
553 atomic_add_return((SUBBUF_OFFSET((offset_old
-1), ltt_buf
)+1),
554 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_old
-1), ltt_buf
)]);
556 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_old
-1),
558 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_old
-1), ltt_buf
),
564 /* Enable signals : this is what guaranteed that same reserve which did the
565 * sem_wait does in fact win the cmpxchg for the offset. We only call
566 * these system calls on buffer boundaries because of their performance
568 if(reserve_commit_diff
== 0) {
569 ret
= pthread_sigmask(SIG_SETMASK
, &oldset
, NULL
);
570 if(ret
) perror("LTT Error in pthread_sigmask\n");
573 /* This code can be executed unordered : writers may already have written
574 to the sub-buffer before this code gets executed, caution. */
575 /* The commit makes sure that this code is executed before the deliver
576 of this sub-buffer */
577 ltt_buffer_begin_callback(ltt_buf
, *tsc
, SUBBUF_INDEX(offset_begin
, ltt_buf
));
578 commit_count
= atomic_add_return(ltt_subbuf_header_len(ltt_buf
),
579 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
580 /* Check if the written buffer has to be delivered */
582 == atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
584 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
588 if(end_switch_current
) {
589 /* current subbuffer */
590 /* Concurrency safe because we are the last and only thread to alter this
591 sub-buffer. As long as it is not delivered and read, no other thread can
592 alter the offset, alter the reserve_count or call the
593 client_buffer_end_callback on this sub-buffer.
594 The only remaining threads could be the ones with pending commits. They
595 will have to do the deliver themself.
596 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
597 with commit and reserve counts. We keep a corrupted sub-buffers count
598 and push the readers across these sub-buffers.
599 Not concurrency safe if a writer is stalled in a subbuffer and
600 another writer switches in, finding out it's corrupted. The result will
601 be than the old (uncommited) subbuffer will be declared corrupted, and
602 that the new subbuffer will be declared corrupted too because of the
603 commit count adjustment. */
604 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_end
,
605 SUBBUF_INDEX((offset_end
-1), ltt_buf
));
606 /* Setting this reserve_count will allow the sub-buffer to be delivered by
607 the last committer. */
609 atomic_add_return((SUBBUF_OFFSET((offset_end
-1), ltt_buf
)+1),
610 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_end
-1), ltt_buf
)]);
612 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_end
-1),
614 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_end
-1), ltt_buf
), NULL
);
620 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
621 //BUG_ON(*slot_size != (offset_end - offset_begin));
623 return ltt_buf
->start
+ BUFFER_OFFSET(offset_begin
, ltt_buf
);
629 * Atomic unordered slot commit. Increments the commit count in the
630 * specified sub-buffer, and delivers it if necessary.
634 * @buf : the buffer to commit to.
635 * @reserved : address of the beginnig of the reserved slot.
636 * @slot_size : size of the reserved slot.
639 static inline void __attribute__((no_instrument_function
)) ltt_commit_slot(
640 struct ltt_buf
*ltt_buf
,
642 unsigned int slot_size
)
644 unsigned int offset_begin
= reserved
- ltt_buf
->start
;
647 commit_count
= atomic_add_return(slot_size
,
648 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
651 /* Check if all commits have been done */
653 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)])) {
654 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
659 } /* end of extern "C" */
662 #endif //LTT_TRACE_FAST
664 #endif //_LTT_USERTRACE_FAST_H