projects / lttv.git / blob
? search:


318eff6a0df7f29730addd11669cb2437e5c02e9
[lttv.git] / usertrace-fast / ltt / ltt-usertrace-fast.h
1
2 /* LTTng user-space "fast" tracing header
3 *
4 * Copyright 2006 Mathieu Desnoyers
5 *
6 */
7
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
10
11 #ifdef LTT_TRACE
12
13 #include <errno.h>
14 #include <asm/atomic.h>
15 #include <pthread.h>
16 #include <stdint.h>
17 #include <syscall.h>
18 #include <asm/timex.h>
19 #include <semaphore.h>
20
21 #include <ltt/ltt-facility-id-user_generic.h>
22 #include <ltt/ltt-generic.h>
23
24 #ifndef LTT_N_SUBBUFS
25 #define LTT_N_SUBBUFS 2
26 #endif //LTT_N_SUBBUFS
27
28 #ifndef LTT_SUBBUF_SIZE_PROCESS
29 #define LTT_SUBBUF_SIZE_PROCESS 1048576
30 #endif //LTT_BUF_SIZE_CPU
31
32 #define LTT_BUF_SIZE_PROCESS (LTT_SUBBUF_SIZE_PROCESS * LTT_N_SUBBUFS)
33
34 #ifndef LTT_USERTRACE_ROOT
35 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
36 #endif //LTT_USERTRACE_ROOT
37
38
39 /* Buffer offset macros */
40
41 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
42 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
43 #define SUBBUF_ALIGN(offset, buf) \
44 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
45 #define SUBBUF_TRUNC(offset, buf) \
46 ((offset) & (~(buf->subbuf_size-1)))
47 #define SUBBUF_INDEX(offset, buf) \
48 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
49
50
51 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
52 #define LTT_TRACER_VERSION_MAJOR 0
53 #define LTT_TRACER_VERSION_MINOR 7
54
55 #ifndef atomic_cmpxchg
56 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
57 #endif //atomic_cmpxchg
58
59 typedef unsigned int ltt_facility_t;
60
61 struct ltt_trace_header {
62 uint32_t magic_number;
63 uint32_t arch_type;
64 uint32_t arch_variant;
65 uint32_t float_word_order; /* Only useful for user space traces */
66 uint8_t arch_size;
67 //uint32_t system_type;
68 uint8_t major_version;
69 uint8_t minor_version;
70 uint8_t flight_recorder;
71 uint8_t has_heartbeat;
72 uint8_t has_alignment; /* Event header alignment */
73 uint32_t freq_scale;
74 uint64_t start_freq;
75 uint64_t start_tsc;
76 uint64_t start_monotonic;
77 uint64_t start_time_sec;
78 uint64_t start_time_usec;
79 } __attribute((packed));
80
81
82 struct ltt_block_start_header {
83 struct {
84 uint64_t cycle_count;
85 uint64_t freq; /* khz */
86 } begin;
87 struct {
88 uint64_t cycle_count;
89 uint64_t freq; /* khz */
90 } end;
91 uint32_t lost_size; /* Size unused at the end of the buffer */
92 uint32_t buf_size; /* The size of this sub-buffer */
93 struct ltt_trace_header trace;
94 } __attribute((packed));
95
96
97
98 struct ltt_buf {
99 void *start;
100 atomic_t offset;
101 atomic_t consumed;
102 atomic_t reserve_count[LTT_N_SUBBUFS];
103 atomic_t commit_count[LTT_N_SUBBUFS];
104
105 atomic_t events_lost;
106 atomic_t corrupted_subbuffers;
107 sem_t writer_sem; /* semaphore on which the writer waits */
108 unsigned int alloc_size;
109 unsigned int subbuf_size;
110 };
111
112 struct ltt_trace_info {
113 int init;
114 int filter;
115 pid_t daemon_id;
116 int nesting;
117 struct {
118 struct ltt_buf process;
119 char process_buf[LTT_BUF_SIZE_PROCESS] __attribute__ ((aligned (8)));
120 } channel;
121 };
122
123
124 struct ltt_event_header_nohb {
125 uint64_t timestamp;
126 unsigned char facility_id;
127 unsigned char event_id;
128 uint16_t event_size;
129 } __attribute((packed));
130
131 extern __thread struct ltt_trace_info *thread_trace_info;
132
133 void ltt_thread_init(void);
134
135 void __attribute__((no_instrument_function))
136 ltt_usertrace_fast_buffer_switch(void);
137
138 /* Get the offset of the channel in the ltt_trace_struct */
139 #define GET_CHANNEL_INDEX(chan) \
140 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
141
142 /* ltt_get_index_from_facility
143 *
144 * Get channel index from facility and event id.
145 *
146 * @fID : facility ID
147 * @eID : event number
148 *
149 * Get the channel index into which events must be written for the given
150 * facility and event number. We get this structure offset as soon as possible
151 * and remember it so we pass through this logic only once per trace call (not
152 * for every trace).
153 */
154 static inline unsigned int __attribute__((no_instrument_function))
155 ltt_get_index_from_facility(ltt_facility_t fID,
156 uint8_t eID)
157 {
158 return GET_CHANNEL_INDEX(process);
159 }
160
161
162 static inline struct ltt_buf * __attribute__((no_instrument_function))
163 ltt_get_channel_from_index(
164 struct ltt_trace_info *trace, unsigned int index)
165 {
166 return (struct ltt_buf *)((void*)trace+index);
167 }
168
169
170 /*
171 * ltt_get_header_size
172 *
173 * Calculate alignment offset for arch size void*. This is the
174 * alignment offset of the event header.
175 *
176 * Important note :
177 * The event header must be a size multiple of the void* size. This is necessary
178 * to be able to calculate statically the alignment offset of the variable
179 * length data fields that follows. The total offset calculated here :
180 *
181 * Alignment of header struct on arch size
182 * + sizeof(header struct)
183 * + padding added to end of struct to align on arch size.
184 * */
185 static inline unsigned char __attribute__((no_instrument_function))
186 ltt_get_header_size(struct ltt_trace_info *trace,
187 void *address,
188 size_t *before_hdr_pad,
189 size_t *after_hdr_pad,
190 size_t *header_size)
191 {
192 unsigned int padding;
193 unsigned int header;
194
195 header = sizeof(struct ltt_event_header_nohb);
196
197 /* Padding before the header. Calculated dynamically */
198 *before_hdr_pad = ltt_align((unsigned long)address, header);
199 padding = *before_hdr_pad;
200
201 /* Padding after header, considering header aligned on ltt_align.
202 * Calculated statically if header size if known. */
203 *after_hdr_pad = ltt_align(header, sizeof(void*));
204 padding += *after_hdr_pad;
205
206 *header_size = header;
207
208 return header+padding;
209 }
210
211
212 /* ltt_write_event_header
213 *
214 * Writes the event header to the pointer.
215 *
216 * @channel : pointer to the channel structure
217 * @ptr : buffer pointer
218 * @fID : facility ID
219 * @eID : event ID
220 * @event_size : size of the event, excluding the event header.
221 * @offset : offset of the beginning of the header, for alignment.
222 * Calculated by ltt_get_event_header_size.
223 * @tsc : time stamp counter.
224 */
225 static inline void __attribute__((no_instrument_function))
226 ltt_write_event_header(
227 struct ltt_trace_info *trace, struct ltt_buf *buf,
228 void *ptr, ltt_facility_t fID, uint32_t eID, size_t event_size,
229 size_t offset, uint64_t tsc)
230 {
231 struct ltt_event_header_nohb *nohb;
232
233 event_size = min(event_size, 0xFFFFU);
234 nohb = (struct ltt_event_header_nohb *)(ptr+offset);
235 nohb->timestamp = (uint64_t)tsc;
236 nohb->facility_id = fID;
237 nohb->event_id = eID;
238 nohb->event_size = (uint16_t)event_size;
239 }
240
241
242
243 static inline uint64_t __attribute__((no_instrument_function))
244 ltt_get_timestamp()
245 {
246 return get_cycles();
247 }
248
249 static inline unsigned int __attribute__((no_instrument_function))
250 ltt_subbuf_header_len(struct ltt_buf *buf)
251 {
252 return sizeof(struct ltt_block_start_header);
253 }
254
255
256
257 static inline void __attribute__((no_instrument_function))
258 ltt_write_trace_header(struct ltt_trace_header *header)
259 {
260 header->magic_number = LTT_TRACER_MAGIC_NUMBER;
261 header->major_version = LTT_TRACER_VERSION_MAJOR;
262 header->minor_version = LTT_TRACER_VERSION_MINOR;
263 header->float_word_order = 0; //FIXME
264 header->arch_type = 0; //FIXME LTT_ARCH_TYPE;
265 header->arch_size = sizeof(void*);
266 header->arch_variant = 0; //FIXME LTT_ARCH_VARIANT;
267 header->flight_recorder = 0;
268 header->has_heartbeat = 0;
269
270 #ifndef LTT_PACK
271 header->has_alignment = sizeof(void*);
272 #else
273 header->has_alignment = 0;
274 #endif
275
276 //FIXME
277 header->freq_scale = 0;
278 header->start_freq = 0;
279 header->start_tsc = 0;
280 header->start_monotonic = 0;
281 header->start_time_sec = 0;
282 header->start_time_usec = 0;
283 }
284
285
286 static inline void __attribute__((no_instrument_function))
287 ltt_buffer_begin_callback(struct ltt_buf *buf,
288 uint64_t tsc, unsigned int subbuf_idx)
289 {
290 struct ltt_block_start_header *header =
291 (struct ltt_block_start_header*)
292 (buf->start + (subbuf_idx*buf->subbuf_size));
293
294 header->begin.cycle_count = tsc;
295 header->begin.freq = 0; //ltt_frequency();
296
297 header->lost_size = 0xFFFFFFFF; // for debugging...
298
299 header->buf_size = buf->subbuf_size;
300
301 ltt_write_trace_header(&header->trace);
302
303 }
304
305
306
307 static inline void __attribute__((no_instrument_function))
308 ltt_buffer_end_callback(struct ltt_buf *buf,
309 uint64_t tsc, unsigned int offset, unsigned int subbuf_idx)
310 {
311 struct ltt_block_start_header *header =
312 (struct ltt_block_start_header*)
313 (buf->start + (subbuf_idx*buf->subbuf_size));
314 /* offset is assumed to never be 0 here : never deliver a completely
315 * empty subbuffer. */
316 /* The lost size is between 0 and subbuf_size-1 */
317 header->lost_size = SUBBUF_OFFSET((buf->subbuf_size - offset),
318 buf);
319 header->end.cycle_count = tsc;
320 header->end.freq = 0; //ltt_frequency();
321 }
322
323
324 static inline void __attribute__((no_instrument_function))
325 ltt_deliver_callback(struct ltt_buf *buf,
326 unsigned subbuf_idx,
327 void *subbuf)
328 {
329 ltt_usertrace_fast_buffer_switch();
330 }
331
332
333 /* ltt_reserve_slot
334 *
335 * Atomic slot reservation in a LTTng buffer. It will take care of
336 * sub-buffer switching.
337 *
338 * Parameters:
339 *
340 * @trace : the trace structure to log to.
341 * @buf : the buffer to reserve space into.
342 * @data_size : size of the variable length data to log.
343 * @slot_size : pointer to total size of the slot (out)
344 * @tsc : pointer to the tsc at the slot reservation (out)
345 * @before_hdr_pad : dynamic padding before the event header.
346 * @after_hdr_pad : dynamic padding after the event header.
347 *
348 * Return : NULL if not enough space, else returns the pointer
349 * to the beginning of the reserved slot. */
350 static inline void * __attribute__((no_instrument_function)) ltt_reserve_slot(
351 struct ltt_trace_info *trace,
352 struct ltt_buf *ltt_buf,
353 unsigned int data_size,
354 unsigned int *slot_size,
355 uint64_t *tsc,
356 size_t *before_hdr_pad,
357 size_t *after_hdr_pad,
358 size_t *header_size)
359 {
360 int offset_begin, offset_end, offset_old;
361 //int has_switch;
362 int begin_switch, end_switch_current, end_switch_old;
363 int reserve_commit_diff = 0;
364 unsigned int size;
365 int consumed_old, consumed_new;
366 int commit_count, reserve_count;
367 int ret;
368
369 do {
370 offset_old = atomic_read(&ltt_buf->offset);
371 offset_begin = offset_old;
372 //has_switch = 0;
373 begin_switch = 0;
374 end_switch_current = 0;
375 end_switch_old = 0;
376 *tsc = ltt_get_timestamp();
377 if(*tsc == 0) {
378 /* Error in getting the timestamp, event lost */
379 atomic_inc(&ltt_buf->events_lost);
380 return NULL;
381 }
382
383 if(SUBBUF_OFFSET(offset_begin, ltt_buf) == 0) {
384 begin_switch = 1; /* For offset_begin */
385 } else {
386 size = ltt_get_header_size(trace, ltt_buf->start + offset_begin,
387 before_hdr_pad, after_hdr_pad, header_size)
388 + data_size;
389
390 if((SUBBUF_OFFSET(offset_begin, ltt_buf)+size)>ltt_buf->subbuf_size) {
391 //has_switch = 1;
392 end_switch_old = 1; /* For offset_old */
393 begin_switch = 1; /* For offset_begin */
394 }
395 }
396
397 if(begin_switch) {
398 if(end_switch_old) {
399 offset_begin = SUBBUF_ALIGN(offset_begin, ltt_buf);
400 }
401 offset_begin = offset_begin + ltt_subbuf_header_len(ltt_buf);
402 /* Test new buffer integrity */
403 reserve_commit_diff =
404 atomic_read(&ltt_buf->reserve_count[SUBBUF_INDEX(offset_begin,
405 ltt_buf)])
406 - atomic_read(&ltt_buf->commit_count[SUBBUF_INDEX(offset_begin,
407 ltt_buf)]);
408 if(reserve_commit_diff == 0) {
409 /* Next buffer not corrupted. */
410 //if((SUBBUF_TRUNC(offset_begin, ltt_buf)
411 // - SUBBUF_TRUNC(atomic_read(&ltt_buf->consumed), ltt_buf))
412 // >= ltt_buf->alloc_size) {
413 sem_wait(&ltt_buf->writer_sem);
414 /* go on with the write */
415
416 //} else {
417 // /* next buffer not corrupted, we are either in overwrite mode or
418 // * the buffer is not full. It's safe to write in this new subbuffer.*/
419 //}
420 } else {
421 /* Next subbuffer corrupted. Force pushing reader even in normal
422 * mode. It's safe to write in this new subbuffer. */
423 sem_post(&ltt_buf->writer_sem);
424 }
425 size = ltt_get_header_size(trace, ltt_buf->start + offset_begin,
426 before_hdr_pad, after_hdr_pad, header_size) + data_size;
427 if((SUBBUF_OFFSET(offset_begin,ltt_buf)+size)>ltt_buf->subbuf_size) {
428 /* Event too big for subbuffers, report error, don't complete
429 * the sub-buffer switch. */
430 atomic_inc(&ltt_buf->events_lost);
431 return NULL;
432 } else {
433 /* We just made a successful buffer switch and the event fits in the
434 * new subbuffer. Let's write. */
435 }
436 } else {
437 /* Event fits in the current buffer and we are not on a switch boundary.
438 * It's safe to write */
439 }
440 offset_end = offset_begin + size;
441
442 if((SUBBUF_OFFSET(offset_end, ltt_buf)) == 0) {
443 /* The offset_end will fall at the very beginning of the next subbuffer.
444 */
445 end_switch_current = 1; /* For offset_begin */
446 }
447
448 } while(atomic_cmpxchg(&ltt_buf->offset, offset_old, offset_end)
449 != offset_old);
450
451
452 /* Push the reader if necessary */
453 do {
454 consumed_old = atomic_read(&ltt_buf->consumed);
455 /* If buffer is in overwrite mode, push the reader consumed count if
456 the write position has reached it and we are not at the first
457 iteration (don't push the reader farther than the writer).
458 This operation can be done concurrently by many writers in the
459 same buffer, the writer being at the fartest write position sub-buffer
460 index in the buffer being the one which will win this loop. */
461 /* If the buffer is not in overwrite mode, pushing the reader only
462 happen if a sub-buffer is corrupted */
463 if((SUBBUF_TRUNC(offset_end, ltt_buf)
464 - SUBBUF_TRUNC(consumed_old, ltt_buf))
465 >= ltt_buf->alloc_size)
466 consumed_new = SUBBUF_ALIGN(consumed_old, ltt_buf);
467 else {
468 consumed_new = consumed_old;
469 break;
470 }
471 } while(atomic_cmpxchg(&ltt_buf->consumed, consumed_old, consumed_new)
472 != consumed_old);
473
474 if(consumed_old != consumed_new) {
475 /* Reader pushed : we are the winner of the push, we can therefore
476 reequilibrate reserve and commit. Atomic increment of the commit
477 count permits other writers to play around with this variable
478 before us. We keep track of corrupted_subbuffers even in overwrite mode :
479 we never want to write over a non completely committed sub-buffer :
480 possible causes : the buffer size is too low compared to the unordered
481 data input, or there is a writer who died between the reserve and the
482 commit. */
483 if(reserve_commit_diff) {
484 /* We have to alter the sub-buffer commit count : a sub-buffer is
485 corrupted. We do not deliver it. */
486 atomic_add(reserve_commit_diff,
487 &ltt_buf->commit_count[SUBBUF_INDEX(offset_begin, ltt_buf)]);
488 atomic_inc(&ltt_buf->corrupted_subbuffers);
489 }
490 }
491
492
493 if(end_switch_old) {
494 /* old subbuffer */
495 /* Concurrency safe because we are the last and only thread to alter this
496 sub-buffer. As long as it is not delivered and read, no other thread can
497 alter the offset, alter the reserve_count or call the
498 client_buffer_end_callback on this sub-buffer.
499 The only remaining threads could be the ones with pending commits. They
500 will have to do the deliver themself.
501 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
502 with commit and reserve counts. We keep a corrupted sub-buffers count
503 and push the readers across these sub-buffers.
504 Not concurrency safe if a writer is stalled in a subbuffer and
505 another writer switches in, finding out it's corrupted. The result will
506 be than the old (uncommited) subbuffer will be declared corrupted, and
507 that the new subbuffer will be declared corrupted too because of the
508 commit count adjustment.
509 Note : offset_old should never be 0 here.*/
510 ltt_buffer_end_callback(ltt_buf, *tsc, offset_old,
511 SUBBUF_INDEX((offset_old-1), ltt_buf));
512 /* Setting this reserve_count will allow the sub-buffer to be delivered by
513 the last committer. */
514 reserve_count =
515 atomic_add_return((SUBBUF_OFFSET((offset_old-1), ltt_buf)+1),
516 &ltt_buf->reserve_count[SUBBUF_INDEX((offset_old-1), ltt_buf)]);
517 if(reserve_count
518 == atomic_read(&ltt_buf->commit_count[SUBBUF_INDEX((offset_old-1),
519 ltt_buf)])) {
520 ltt_deliver_callback(ltt_buf, SUBBUF_INDEX((offset_old-1), ltt_buf),
521 NULL);
522 }
523 }
524
525 if(begin_switch) {
526 /* New sub-buffer */
527 /* This code can be executed unordered : writers may already have written
528 to the sub-buffer before this code gets executed, caution. */
529 /* The commit makes sure that this code is executed before the deliver
530 of this sub-buffer */
531 ltt_buffer_begin_callback(ltt_buf, *tsc, SUBBUF_INDEX(offset_begin, ltt_buf));
532 commit_count = atomic_add_return(ltt_subbuf_header_len(ltt_buf),
533 &ltt_buf->commit_count[SUBBUF_INDEX(offset_begin, ltt_buf)]);
534 /* Check if the written buffer has to be delivered */
535 if(commit_count
536 == atomic_read(&ltt_buf->reserve_count[SUBBUF_INDEX(offset_begin,
537 ltt_buf)])) {
538 ltt_deliver_callback(ltt_buf, SUBBUF_INDEX(offset_begin, ltt_buf), NULL);
539 }
540 }
541
542 if(end_switch_current) {
543 /* current subbuffer */
544 /* Concurrency safe because we are the last and only thread to alter this
545 sub-buffer. As long as it is not delivered and read, no other thread can
546 alter the offset, alter the reserve_count or call the
547 client_buffer_end_callback on this sub-buffer.
548 The only remaining threads could be the ones with pending commits. They
549 will have to do the deliver themself.
550 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
551 with commit and reserve counts. We keep a corrupted sub-buffers count
552 and push the readers across these sub-buffers.
553 Not concurrency safe if a writer is stalled in a subbuffer and
554 another writer switches in, finding out it's corrupted. The result will
555 be than the old (uncommited) subbuffer will be declared corrupted, and
556 that the new subbuffer will be declared corrupted too because of the
557 commit count adjustment. */
558 ltt_buffer_end_callback(ltt_buf, *tsc, offset_end,
559 SUBBUF_INDEX((offset_end-1), ltt_buf));
560 /* Setting this reserve_count will allow the sub-buffer to be delivered by
561 the last committer. */
562 reserve_count =
563 atomic_add_return((SUBBUF_OFFSET((offset_end-1), ltt_buf)+1),
564 &ltt_buf->reserve_count[SUBBUF_INDEX((offset_end-1), ltt_buf)]);
565 if(reserve_count
566 == atomic_read(&ltt_buf->commit_count[SUBBUF_INDEX((offset_end-1),
567 ltt_buf)])) {
568 ltt_deliver_callback(ltt_buf, SUBBUF_INDEX((offset_end-1), ltt_buf), NULL);
569 }
570 }
571
572 *slot_size = size;
573
574 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
575 //BUG_ON(*slot_size != (offset_end - offset_begin));
576
577 return ltt_buf->start + BUFFER_OFFSET(offset_begin, ltt_buf);
578 }
579
580
581 /* ltt_commit_slot
582 *
583 * Atomic unordered slot commit. Increments the commit count in the
584 * specified sub-buffer, and delivers it if necessary.
585 *
586 * Parameters:
587 *
588 * @buf : the buffer to commit to.
589 * @reserved : address of the beginnig of the reserved slot.
590 * @slot_size : size of the reserved slot.
591 *
592 */
593 static inline void __attribute__((no_instrument_function)) ltt_commit_slot(
594 struct ltt_buf *ltt_buf,
595 void *reserved,
596 unsigned int slot_size)
597 {
598 unsigned int offset_begin = reserved - ltt_buf->start;
599 int commit_count;
600
601 commit_count = atomic_add_return(slot_size,
602 &ltt_buf->commit_count[SUBBUF_INDEX(offset_begin,
603 ltt_buf)]);
604
605 /* Check if all commits have been done */
606 if(commit_count ==
607 atomic_read(&ltt_buf->reserve_count[SUBBUF_INDEX(offset_begin, ltt_buf)])) {
608 ltt_deliver_callback(ltt_buf, SUBBUF_INDEX(offset_begin, ltt_buf), NULL);
609 }
610 }
611
612
613 #endif //LTT_TRACE
614
615
616 #endif //_LTT_USERTRACE_FAST_H
LTTng 2.x Viewer (LTTV)
This page took 0.040033 seconds and 3 git commands to generate.