2 /* LTTng user-space "fast" tracing header
4 * Copyright 2006 Mathieu Desnoyers
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
18 #include <asm/timex.h>
19 #include <semaphore.h>
22 #define __KERNEL__ /* Ugly hack : atomic.h is broken */
24 #include <asm/atomic.h>
26 #include "ltt/ltt-usertrace-ppc.h"
27 #undef __KERNEL__ /* Ugly hack : atomic.h is broken */
30 #include <ltt/ltt-facility-id-user_generic.h>
33 #define LTT_N_SUBBUFS 2
34 #endif //LTT_N_SUBBUFS
36 #ifndef LTT_SUBBUF_SIZE_PROCESS
37 #define LTT_SUBBUF_SIZE_PROCESS 1048576
38 #endif //LTT_BUF_SIZE_CPU
40 #define LTT_BUF_SIZE_PROCESS (LTT_SUBBUF_SIZE_PROCESS * LTT_N_SUBBUFS)
42 #ifndef LTT_USERTRACE_ROOT
43 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
44 #endif //LTT_USERTRACE_ROOT
47 /* Buffer offset macros */
49 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
50 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
51 #define SUBBUF_ALIGN(offset, buf) \
52 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
53 #define SUBBUF_TRUNC(offset, buf) \
54 ((offset) & (~(buf->subbuf_size-1)))
55 #define SUBBUF_INDEX(offset, buf) \
56 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
59 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
60 #define LTT_TRACER_VERSION_MAJOR 0
61 #define LTT_TRACER_VERSION_MINOR 7
63 #ifndef atomic_cmpxchg
64 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
65 #endif //atomic_cmpxchg
67 struct ltt_trace_header
{
68 uint32_t magic_number
;
70 uint32_t arch_variant
;
71 uint32_t float_word_order
; /* Only useful for user space traces */
73 //uint32_t system_type;
74 uint8_t major_version
;
75 uint8_t minor_version
;
76 uint8_t flight_recorder
;
77 uint8_t has_heartbeat
;
78 uint8_t has_alignment
; /* Event header alignment */
82 uint64_t start_monotonic
;
83 uint64_t start_time_sec
;
84 uint64_t start_time_usec
;
85 } __attribute((packed
));
88 struct ltt_block_start_header
{
91 uint64_t freq
; /* khz */
95 uint64_t freq
; /* khz */
97 uint32_t lost_size
; /* Size unused at the end of the buffer */
98 uint32_t buf_size
; /* The size of this sub-buffer */
99 struct ltt_trace_header trace
;
100 } __attribute((packed
));
108 atomic_t reserve_count
[LTT_N_SUBBUFS
];
109 atomic_t commit_count
[LTT_N_SUBBUFS
];
111 atomic_t events_lost
;
112 atomic_t corrupted_subbuffers
;
113 sem_t writer_sem
; /* semaphore on which the writer waits */
114 unsigned int alloc_size
;
115 unsigned int subbuf_size
;
118 struct ltt_trace_info
{
124 struct ltt_buf process
;
125 char process_buf
[LTT_BUF_SIZE_PROCESS
] __attribute__ ((aligned (8)));
130 struct ltt_event_header_nohb
{
132 unsigned char facility_id
;
133 unsigned char event_id
;
135 } __attribute((packed
));
137 extern __thread
struct ltt_trace_info
*thread_trace_info
;
139 void ltt_thread_init(void);
141 void __attribute__((no_instrument_function
))
142 ltt_usertrace_fast_buffer_switch(void);
144 /* Get the offset of the channel in the ltt_trace_struct */
145 #define GET_CHANNEL_INDEX(chan) \
146 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
148 /* ltt_get_index_from_facility
150 * Get channel index from facility and event id.
153 * @eID : event number
155 * Get the channel index into which events must be written for the given
156 * facility and event number. We get this structure offset as soon as possible
157 * and remember it so we pass through this logic only once per trace call (not
160 static inline unsigned int __attribute__((no_instrument_function
))
161 ltt_get_index_from_facility(ltt_facility_t fID
,
164 return GET_CHANNEL_INDEX(process
);
168 static inline struct ltt_buf
* __attribute__((no_instrument_function
))
169 ltt_get_channel_from_index(
170 struct ltt_trace_info
*trace
, unsigned int index
)
172 return (struct ltt_buf
*)((void*)trace
+index
);
177 * ltt_get_header_size
179 * Calculate alignment offset for arch size void*. This is the
180 * alignment offset of the event header.
183 * The event header must be a size multiple of the void* size. This is necessary
184 * to be able to calculate statically the alignment offset of the variable
185 * length data fields that follows. The total offset calculated here :
187 * Alignment of header struct on arch size
188 * + sizeof(header struct)
189 * + padding added to end of struct to align on arch size.
191 static inline unsigned char __attribute__((no_instrument_function
))
192 ltt_get_header_size(struct ltt_trace_info
*trace
,
194 size_t *before_hdr_pad
,
195 size_t *after_hdr_pad
,
198 unsigned int padding
;
201 header
= sizeof(struct ltt_event_header_nohb
);
203 /* Padding before the header. Calculated dynamically */
204 *before_hdr_pad
= ltt_align((unsigned long)address
, header
);
205 padding
= *before_hdr_pad
;
207 /* Padding after header, considering header aligned on ltt_align.
208 * Calculated statically if header size if known. */
209 *after_hdr_pad
= ltt_align(header
, sizeof(void*));
210 padding
+= *after_hdr_pad
;
212 *header_size
= header
;
214 return header
+padding
;
218 /* ltt_write_event_header
220 * Writes the event header to the pointer.
222 * @channel : pointer to the channel structure
223 * @ptr : buffer pointer
226 * @event_size : size of the event, excluding the event header.
227 * @offset : offset of the beginning of the header, for alignment.
228 * Calculated by ltt_get_event_header_size.
229 * @tsc : time stamp counter.
231 static inline void __attribute__((no_instrument_function
))
232 ltt_write_event_header(
233 struct ltt_trace_info
*trace
, struct ltt_buf
*buf
,
234 void *ptr
, ltt_facility_t fID
, uint32_t eID
, size_t event_size
,
235 size_t offset
, uint64_t tsc
)
237 struct ltt_event_header_nohb
*nohb
;
239 event_size
= min(event_size
, 0xFFFFU
);
240 nohb
= (struct ltt_event_header_nohb
*)(ptr
+offset
);
241 nohb
->timestamp
= (uint64_t)tsc
;
242 nohb
->facility_id
= fID
;
243 nohb
->event_id
= eID
;
244 nohb
->event_size
= (uint16_t)event_size
;
249 static inline uint64_t __attribute__((no_instrument_function
))
255 static inline unsigned int __attribute__((no_instrument_function
))
256 ltt_subbuf_header_len(struct ltt_buf
*buf
)
258 return sizeof(struct ltt_block_start_header
);
263 static inline void __attribute__((no_instrument_function
))
264 ltt_write_trace_header(struct ltt_trace_header
*header
)
266 header
->magic_number
= LTT_TRACER_MAGIC_NUMBER
;
267 header
->major_version
= LTT_TRACER_VERSION_MAJOR
;
268 header
->minor_version
= LTT_TRACER_VERSION_MINOR
;
269 header
->float_word_order
= 0; //FIXME
270 header
->arch_type
= 0; //FIXME LTT_ARCH_TYPE;
271 header
->arch_size
= sizeof(void*);
272 header
->arch_variant
= 0; //FIXME LTT_ARCH_VARIANT;
273 header
->flight_recorder
= 0;
274 header
->has_heartbeat
= 0;
277 header
->has_alignment
= sizeof(void*);
279 header
->has_alignment
= 0;
283 header
->freq_scale
= 0;
284 header
->start_freq
= 0;
285 header
->start_tsc
= 0;
286 header
->start_monotonic
= 0;
287 header
->start_time_sec
= 0;
288 header
->start_time_usec
= 0;
292 static inline void __attribute__((no_instrument_function
))
293 ltt_buffer_begin_callback(struct ltt_buf
*buf
,
294 uint64_t tsc
, unsigned int subbuf_idx
)
296 struct ltt_block_start_header
*header
=
297 (struct ltt_block_start_header
*)
298 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
300 header
->begin
.cycle_count
= tsc
;
301 header
->begin
.freq
= 0; //ltt_frequency();
303 header
->lost_size
= 0xFFFFFFFF; // for debugging...
305 header
->buf_size
= buf
->subbuf_size
;
307 ltt_write_trace_header(&header
->trace
);
313 static inline void __attribute__((no_instrument_function
))
314 ltt_buffer_end_callback(struct ltt_buf
*buf
,
315 uint64_t tsc
, unsigned int offset
, unsigned int subbuf_idx
)
317 struct ltt_block_start_header
*header
=
318 (struct ltt_block_start_header
*)
319 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
320 /* offset is assumed to never be 0 here : never deliver a completely
321 * empty subbuffer. */
322 /* The lost size is between 0 and subbuf_size-1 */
323 header
->lost_size
= SUBBUF_OFFSET((buf
->subbuf_size
- offset
),
325 header
->end
.cycle_count
= tsc
;
326 header
->end
.freq
= 0; //ltt_frequency();
330 static inline void __attribute__((no_instrument_function
))
331 ltt_deliver_callback(struct ltt_buf
*buf
,
335 ltt_usertrace_fast_buffer_switch();
341 * Atomic slot reservation in a LTTng buffer. It will take care of
342 * sub-buffer switching.
346 * @trace : the trace structure to log to.
347 * @buf : the buffer to reserve space into.
348 * @data_size : size of the variable length data to log.
349 * @slot_size : pointer to total size of the slot (out)
350 * @tsc : pointer to the tsc at the slot reservation (out)
351 * @before_hdr_pad : dynamic padding before the event header.
352 * @after_hdr_pad : dynamic padding after the event header.
354 * Return : NULL if not enough space, else returns the pointer
355 * to the beginning of the reserved slot. */
356 static inline void * __attribute__((no_instrument_function
)) ltt_reserve_slot(
357 struct ltt_trace_info
*trace
,
358 struct ltt_buf
*ltt_buf
,
359 unsigned int data_size
,
362 size_t *before_hdr_pad
,
363 size_t *after_hdr_pad
,
366 int offset_begin
, offset_end
, offset_old
;
368 int begin_switch
, end_switch_current
, end_switch_old
;
369 int reserve_commit_diff
= 0;
371 int consumed_old
, consumed_new
;
372 int commit_count
, reserve_count
;
376 offset_old
= atomic_read(<t_buf
->offset
);
377 offset_begin
= offset_old
;
380 end_switch_current
= 0;
382 *tsc
= ltt_get_timestamp();
384 /* Error in getting the timestamp, event lost */
385 atomic_inc(<t_buf
->events_lost
);
389 if(SUBBUF_OFFSET(offset_begin
, ltt_buf
) == 0) {
390 begin_switch
= 1; /* For offset_begin */
392 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
393 before_hdr_pad
, after_hdr_pad
, header_size
)
396 if((SUBBUF_OFFSET(offset_begin
, ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
398 end_switch_old
= 1; /* For offset_old */
399 begin_switch
= 1; /* For offset_begin */
405 offset_begin
= SUBBUF_ALIGN(offset_begin
, ltt_buf
);
407 offset_begin
= offset_begin
+ ltt_subbuf_header_len(ltt_buf
);
408 /* Test new buffer integrity */
409 reserve_commit_diff
=
410 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
412 - atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
414 if(reserve_commit_diff
== 0) {
415 /* Next buffer not corrupted. */
416 //if((SUBBUF_TRUNC(offset_begin, ltt_buf)
417 // - SUBBUF_TRUNC(atomic_read(<t_buf->consumed), ltt_buf))
418 // >= ltt_buf->alloc_size) {
419 /* sem_wait is not signal safe. Disable signals around it. */
421 sigset_t oldset
, set
;
423 /* Disable signals */
424 ret
= sigfillset(&set
);
425 if(ret
) perror("LTT Error in sigfillset\n");
427 ret
= pthread_sigmask(SIG_BLOCK
, &set
, &oldset
);
428 if(ret
) perror("LTT Error in pthread_sigmask\n");
430 sem_wait(<t_buf
->writer_sem
);
433 ret
= pthread_sigmask(SIG_SETMASK
, &oldset
, NULL
);
434 if(ret
) perror("LTT Error in pthread_sigmask\n");
437 /* go on with the write */
440 // /* next buffer not corrupted, we are either in overwrite mode or
441 // * the buffer is not full. It's safe to write in this new subbuffer.*/
444 /* Next subbuffer corrupted. Force pushing reader even in normal
445 * mode. It's safe to write in this new subbuffer. */
446 sem_post(<t_buf
->writer_sem
);
448 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
449 before_hdr_pad
, after_hdr_pad
, header_size
) + data_size
;
450 if((SUBBUF_OFFSET(offset_begin
,ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
451 /* Event too big for subbuffers, report error, don't complete
452 * the sub-buffer switch. */
453 atomic_inc(<t_buf
->events_lost
);
456 /* We just made a successful buffer switch and the event fits in the
457 * new subbuffer. Let's write. */
460 /* Event fits in the current buffer and we are not on a switch boundary.
461 * It's safe to write */
463 offset_end
= offset_begin
+ size
;
465 if((SUBBUF_OFFSET(offset_end
, ltt_buf
)) == 0) {
466 /* The offset_end will fall at the very beginning of the next subbuffer.
468 end_switch_current
= 1; /* For offset_begin */
471 } while(atomic_cmpxchg(<t_buf
->offset
, offset_old
, offset_end
)
475 /* Push the reader if necessary */
477 consumed_old
= atomic_read(<t_buf
->consumed
);
478 /* If buffer is in overwrite mode, push the reader consumed count if
479 the write position has reached it and we are not at the first
480 iteration (don't push the reader farther than the writer).
481 This operation can be done concurrently by many writers in the
482 same buffer, the writer being at the fartest write position sub-buffer
483 index in the buffer being the one which will win this loop. */
484 /* If the buffer is not in overwrite mode, pushing the reader only
485 happen if a sub-buffer is corrupted */
486 if((SUBBUF_TRUNC(offset_end
-1, ltt_buf
)
487 - SUBBUF_TRUNC(consumed_old
, ltt_buf
))
488 >= ltt_buf
->alloc_size
)
489 consumed_new
= SUBBUF_ALIGN(consumed_old
, ltt_buf
);
491 consumed_new
= consumed_old
;
494 } while(atomic_cmpxchg(<t_buf
->consumed
, consumed_old
, consumed_new
)
497 if(consumed_old
!= consumed_new
) {
498 /* Reader pushed : we are the winner of the push, we can therefore
499 reequilibrate reserve and commit. Atomic increment of the commit
500 count permits other writers to play around with this variable
501 before us. We keep track of corrupted_subbuffers even in overwrite mode :
502 we never want to write over a non completely committed sub-buffer :
503 possible causes : the buffer size is too low compared to the unordered
504 data input, or there is a writer who died between the reserve and the
506 if(reserve_commit_diff
) {
507 /* We have to alter the sub-buffer commit count : a sub-buffer is
508 corrupted. We do not deliver it. */
509 atomic_add(reserve_commit_diff
,
510 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
511 atomic_inc(<t_buf
->corrupted_subbuffers
);
518 /* Concurrency safe because we are the last and only thread to alter this
519 sub-buffer. As long as it is not delivered and read, no other thread can
520 alter the offset, alter the reserve_count or call the
521 client_buffer_end_callback on this sub-buffer.
522 The only remaining threads could be the ones with pending commits. They
523 will have to do the deliver themself.
524 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
525 with commit and reserve counts. We keep a corrupted sub-buffers count
526 and push the readers across these sub-buffers.
527 Not concurrency safe if a writer is stalled in a subbuffer and
528 another writer switches in, finding out it's corrupted. The result will
529 be than the old (uncommited) subbuffer will be declared corrupted, and
530 that the new subbuffer will be declared corrupted too because of the
531 commit count adjustment.
532 Note : offset_old should never be 0 here.*/
533 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_old
,
534 SUBBUF_INDEX((offset_old
-1), ltt_buf
));
535 /* Setting this reserve_count will allow the sub-buffer to be delivered by
536 the last committer. */
538 atomic_add_return((SUBBUF_OFFSET((offset_old
-1), ltt_buf
)+1),
539 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_old
-1), ltt_buf
)]);
541 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_old
-1),
543 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_old
-1), ltt_buf
),
550 /* This code can be executed unordered : writers may already have written
551 to the sub-buffer before this code gets executed, caution. */
552 /* The commit makes sure that this code is executed before the deliver
553 of this sub-buffer */
554 ltt_buffer_begin_callback(ltt_buf
, *tsc
, SUBBUF_INDEX(offset_begin
, ltt_buf
));
555 commit_count
= atomic_add_return(ltt_subbuf_header_len(ltt_buf
),
556 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
557 /* Check if the written buffer has to be delivered */
559 == atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
561 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
565 if(end_switch_current
) {
566 /* current subbuffer */
567 /* Concurrency safe because we are the last and only thread to alter this
568 sub-buffer. As long as it is not delivered and read, no other thread can
569 alter the offset, alter the reserve_count or call the
570 client_buffer_end_callback on this sub-buffer.
571 The only remaining threads could be the ones with pending commits. They
572 will have to do the deliver themself.
573 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
574 with commit and reserve counts. We keep a corrupted sub-buffers count
575 and push the readers across these sub-buffers.
576 Not concurrency safe if a writer is stalled in a subbuffer and
577 another writer switches in, finding out it's corrupted. The result will
578 be than the old (uncommited) subbuffer will be declared corrupted, and
579 that the new subbuffer will be declared corrupted too because of the
580 commit count adjustment. */
581 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_end
,
582 SUBBUF_INDEX((offset_end
-1), ltt_buf
));
583 /* Setting this reserve_count will allow the sub-buffer to be delivered by
584 the last committer. */
586 atomic_add_return((SUBBUF_OFFSET((offset_end
-1), ltt_buf
)+1),
587 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_end
-1), ltt_buf
)]);
589 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_end
-1),
591 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_end
-1), ltt_buf
), NULL
);
597 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
598 //BUG_ON(*slot_size != (offset_end - offset_begin));
600 return ltt_buf
->start
+ BUFFER_OFFSET(offset_begin
, ltt_buf
);
606 * Atomic unordered slot commit. Increments the commit count in the
607 * specified sub-buffer, and delivers it if necessary.
611 * @buf : the buffer to commit to.
612 * @reserved : address of the beginnig of the reserved slot.
613 * @slot_size : size of the reserved slot.
616 static inline void __attribute__((no_instrument_function
)) ltt_commit_slot(
617 struct ltt_buf
*ltt_buf
,
619 unsigned int slot_size
)
621 unsigned int offset_begin
= reserved
- ltt_buf
->start
;
624 commit_count
= atomic_add_return(slot_size
,
625 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
628 /* Check if all commits have been done */
630 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)])) {
631 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
636 #endif //LTT_TRACE_FAST
638 #endif //_LTT_USERTRACE_FAST_H