3 * Linux Trace Toolkit Daemon
5 * This is a simple daemon that reads a few relay+debugfs channels and save
8 * CPU hot-plugging is supported using inotify.
11 * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
26 #include <sys/types.h>
34 #include <sys/syscall.h>
36 #include <asm/ioctls.h>
38 #include <linux/version.h>
41 #include <asm/ioctl.h>
42 #include <asm/types.h>
44 /* Get the next sub buffer that can be read. */
45 #define RELAY_GET_SB _IOR(0xF5, 0x00,__u32)
46 /* Release the oldest reserved (by "get") sub buffer. */
47 #define RELAY_PUT_SB _IOW(0xF5, 0x01,__u32)
48 /* returns the number of sub buffers in the per cpu channel. */
49 #define RELAY_GET_N_SB _IOR(0xF5, 0x02,__u32)
50 /* returns the size of the current sub buffer. */
51 #define RELAY_GET_SB_SIZE _IOR(0xF5, 0x03, __u32)
52 /* returns the size of data to consume in the current sub-buffer. */
53 #define RELAY_GET_MAX_SB_SIZE _IOR(0xF5, 0x04, __u32)
56 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14)
57 #include <sys/inotify.h>
61 static inline int inotify_init (void)
66 static inline int inotify_add_watch (int fd
, const char *name
, __u32 mask
)
71 static inline int inotify_rm_watch (int fd
, __u32 wd
)
78 struct channel_trace_fd
{
83 struct inotify_watch
{
85 char path_channel
[PATH_MAX
];
86 char *base_path_channel
;
89 struct inotify_watch_array
{
90 struct inotify_watch
*elem
;
94 struct liblttd_instance
{
95 struct liblttd_callbacks
*callbacks
;
98 struct channel_trace_fd fd_pairs
;
99 struct inotify_watch_array inotify_watch_array
;
101 /* protects fd_pairs and inotify_watch_array */
102 pthread_rwlock_t fd_pairs_lock
;
104 char channel_name
[PATH_MAX
];
105 unsigned long num_threads
;
106 int quit_program
; /* For signal handler */
107 int dump_flight_only
;
108 int dump_normal_only
;
112 struct liblttd_thread_data
{
114 struct liblttd_instance
*instance
;
117 #define printf_verbose(fmt, args...) \
119 if (instance->verbose_mode) \
120 printf(fmt, ##args); \
124 int open_buffer_file(struct liblttd_instance
*instance
, char *filename
,
125 char *path_channel
, char *base_path_channel
)
130 if(strncmp(filename
, "flight-", sizeof("flight-")-1) != 0) {
131 if(instance
->dump_flight_only
) {
132 printf_verbose("Skipping normal channel %s\n",
137 if(instance
->dump_normal_only
) {
138 printf_verbose("Skipping flight channel %s\n",
143 printf_verbose("Opening file.\n");
145 instance
->fd_pairs
.pair
= realloc(instance
->fd_pairs
.pair
,
146 ++instance
->fd_pairs
.num_pairs
* sizeof(struct fd_pair
));
148 /* Open the channel in read mode */
149 instance
->fd_pairs
.pair
[instance
->fd_pairs
.num_pairs
-1].channel
=
150 open(path_channel
, O_RDONLY
| O_NONBLOCK
);
151 if(instance
->fd_pairs
.pair
[instance
->fd_pairs
.num_pairs
-1].channel
== -1) {
152 perror(path_channel
);
153 instance
->fd_pairs
.num_pairs
--;
154 return 0; /* continue */
157 if(instance
->callbacks
->on_open_channel
) ret
= instance
->callbacks
->on_open_channel(
158 instance
->callbacks
, &instance
->fd_pairs
.pair
[instance
->fd_pairs
.num_pairs
-1],
163 close(instance
->fd_pairs
.pair
[instance
->fd_pairs
.num_pairs
-1].channel
);
164 instance
->fd_pairs
.num_pairs
--;
172 int open_channel_trace_pairs(struct liblttd_instance
*instance
,
173 char *subchannel_name
, char *base_subchannel_name
)
175 DIR *channel_dir
= opendir(subchannel_name
);
176 struct dirent
*entry
;
177 struct stat stat_buf
;
179 char path_channel
[PATH_MAX
];
180 int path_channel_len
;
181 char *path_channel_ptr
;
182 char *base_subchannel_ptr
;
186 if(channel_dir
== NULL
) {
187 perror(subchannel_name
);
192 printf_verbose("Calling on new channels folder\n");
193 if(instance
->callbacks
->on_new_channels_folder
) ret
= instance
->callbacks
->
194 on_new_channels_folder(instance
->callbacks
,
195 base_subchannel_name
);
201 strncpy(path_channel
, subchannel_name
, PATH_MAX
-1);
202 path_channel_len
= strlen(path_channel
);
203 path_channel
[path_channel_len
] = '/';
205 path_channel_ptr
= path_channel
+ path_channel_len
;
206 base_subchannel_ptr
= path_channel
+
207 (base_subchannel_name
- subchannel_name
);
210 instance
->inotify_watch_array
.elem
= realloc(instance
->inotify_watch_array
.elem
,
211 ++instance
->inotify_watch_array
.num
* sizeof(struct inotify_watch
));
213 printf_verbose("Adding inotify for channel %s\n", path_channel
);
214 instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].wd
= inotify_add_watch(instance
->inotify_fd
, path_channel
, IN_CREATE
);
215 strcpy(instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].path_channel
, path_channel
);
216 instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].base_path_channel
=
217 instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].path_channel
+
218 (base_subchannel_name
- subchannel_name
);
219 printf_verbose("Added inotify for channel %s, wd %u\n",
220 instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].path_channel
,
221 instance
->inotify_watch_array
.elem
[instance
->inotify_watch_array
.num
-1].wd
);
224 while((entry
= readdir(channel_dir
)) != NULL
) {
226 if(entry
->d_name
[0] == '.') continue;
228 strncpy(path_channel_ptr
, entry
->d_name
, PATH_MAX
- path_channel_len
);
230 ret
= stat(path_channel
, &stat_buf
);
232 perror(path_channel
);
236 printf_verbose("Channel file : %s\n", path_channel
);
238 if(S_ISDIR(stat_buf
.st_mode
)) {
240 printf_verbose("Entering channel subdirectory...\n");
241 ret
= open_channel_trace_pairs(instance
, path_channel
, base_subchannel_ptr
);
242 if(ret
< 0) continue;
243 } else if(S_ISREG(stat_buf
.st_mode
)) {
244 open_ret
= open_buffer_file(instance
, entry
->d_name
,
245 path_channel
, base_subchannel_ptr
);
252 closedir(channel_dir
);
258 int read_subbuffer(struct liblttd_instance
*instance
, struct fd_pair
*pair
)
260 unsigned int consumed_old
, len
;
266 err
= ioctl(pair
->channel
, RELAY_GET_SB
, &consumed_old
);
267 printf_verbose("cookie : %u\n", consumed_old
);
270 perror("Reserving sub buffer failed (everything is normal, it is due to concurrency)");
274 err
= ioctl(pair
->channel
, RELAY_GET_SB_SIZE
, &len
);
277 perror("Getting sub-buffer len failed.");
281 if(instance
->callbacks
->on_read_subbuffer
) ret
= instance
->callbacks
->on_read_subbuffer(
282 instance
->callbacks
, pair
, len
);
286 err
= ioctl(pair
->channel
, RELAY_PUT_SB
, &consumed_old
);
289 if(errno
== EFAULT
) {
290 perror("Error in unreserving sub buffer\n");
291 } else if(errno
== EIO
) {
292 /* Should never happen with newer LTTng versions */
293 perror("Reader has been pushed by the writer, last sub-buffer corrupted.");
303 int map_channels(struct liblttd_instance
*instance
, int idx_begin
, int idx_end
)
308 if(instance
->fd_pairs
.num_pairs
<= 0) {
309 printf("No channel to read\n");
313 /* Get the subbuf sizes and number */
315 for(i
=idx_begin
;i
<idx_end
;i
++) {
316 struct fd_pair
*pair
= &instance
->fd_pairs
.pair
[i
];
318 ret
= ioctl(pair
->channel
, RELAY_GET_N_SB
, &pair
->n_sb
);
320 perror("Error in getting the number of sub-buffers");
323 ret
= ioctl(pair
->channel
, RELAY_GET_MAX_SB_SIZE
,
326 perror("Error in getting the max sub-buffer size");
329 ret
= pthread_mutex_init(&pair
->mutex
, NULL
); /* Fast mutex */
331 perror("Error in mutex init");
340 int unmap_channels(struct liblttd_instance
*instance
)
346 for(j
=0;j
<instance
->fd_pairs
.num_pairs
;j
++) {
347 struct fd_pair
*pair
= &instance
->fd_pairs
.pair
[j
];
350 err_ret
= pthread_mutex_destroy(&pair
->mutex
);
352 perror("Error in mutex destroy");
361 /* Inotify event arrived.
363 * Only support add file for now.
365 int read_inotify(struct liblttd_instance
*instance
)
367 char buf
[sizeof(struct inotify_event
) + PATH_MAX
];
368 char path_channel
[PATH_MAX
];
370 struct inotify_event
*ievent
;
377 len
= read(instance
->inotify_fd
, buf
, sizeof(struct inotify_event
) + PATH_MAX
);
381 return 0; /* another thread got the data before us */
383 printf("Error in read from inotify FD %s.\n", strerror(len
));
386 while(offset
< len
) {
387 ievent
= (struct inotify_event
*)&(buf
[offset
]);
388 for(i
=0; i
<instance
->inotify_watch_array
.num
; i
++) {
389 if(instance
->inotify_watch_array
.elem
[i
].wd
== ievent
->wd
&&
390 ievent
->mask
== IN_CREATE
) {
392 "inotify wd %u event mask : %u for %s%s\n",
393 ievent
->wd
, ievent
->mask
,
394 instance
->inotify_watch_array
.elem
[i
].path_channel
,
396 old_num
= instance
->fd_pairs
.num_pairs
;
397 strcpy(path_channel
, instance
->inotify_watch_array
.elem
[i
].path_channel
);
398 strcat(path_channel
, ievent
->name
);
399 if(ret
= open_buffer_file(instance
, ievent
->name
, path_channel
,
400 path_channel
+ (instance
->inotify_watch_array
.elem
[i
].base_path_channel
-
401 instance
->inotify_watch_array
.elem
[i
].path_channel
))) {
402 printf("Error opening buffer file\n");
405 if(ret
= map_channels(instance
, old_num
, instance
->fd_pairs
.num_pairs
)) {
406 printf("Error mapping channel\n");
412 offset
+= sizeof(*ievent
) + ievent
->len
;
421 * Read the debugfs channels and write them in the paired tracefiles.
423 * @fd_pairs : paired channels and trace files.
425 * returns 0 on success, -1 on error.
427 * Note that the high priority polled channels are consumed first. We then poll
428 * again to see if these channels are still in priority. Only when no
429 * high priority channel is left, we start reading low priority channels.
431 * Note that a channel is considered high priority when the buffer is almost
435 int read_channels(struct liblttd_instance
*instance
, unsigned long thread_num
)
437 struct pollfd
*pollfd
= NULL
;
440 int num_rdy
, num_hup
;
444 unsigned int old_num
;
452 pthread_rwlock_rdlock(&instance
->fd_pairs_lock
);
454 /* Start polling the FD. Keep one fd for inotify */
455 pollfd
= malloc((inotify_fds
+ instance
->fd_pairs
.num_pairs
) * sizeof(struct pollfd
));
458 pollfd
[0].fd
= instance
->inotify_fd
;
459 pollfd
[0].events
= POLLIN
|POLLPRI
;
462 for(i
=0;i
<instance
->fd_pairs
.num_pairs
;i
++) {
463 pollfd
[inotify_fds
+i
].fd
= instance
->fd_pairs
.pair
[i
].channel
;
464 pollfd
[inotify_fds
+i
].events
= POLLIN
|POLLPRI
;
466 num_pollfd
= inotify_fds
+ instance
->fd_pairs
.num_pairs
;
469 pthread_rwlock_unlock(&instance
->fd_pairs_lock
);
475 printf("Press a key for next poll...\n");
477 read(STDIN_FILENO
, &buf
, 1);
478 printf("Next poll (polling %d fd) :\n", num_pollfd
);
481 /* Have we received a signal ? */
482 if(instance
->quit_program
) break;
484 num_rdy
= poll(pollfd
, num_pollfd
, -1);
487 perror("Poll error");
491 printf_verbose("Data received\n");
493 switch(pollfd
[0].revents
) {
496 "Error returned in polling inotify fd %d.\n",
501 "Polling inotify fd %d tells it has hung up.\n",
506 "Polling inotify fd %d tells fd is not open.\n",
512 "Polling inotify fd %d : data ready.\n",
515 pthread_rwlock_wrlock(&instance
->fd_pairs_lock
);
516 read_inotify(instance
);
517 pthread_rwlock_unlock(&instance
->fd_pairs_lock
);
523 for(i
=inotify_fds
;i
<num_pollfd
;i
++) {
524 switch(pollfd
[i
].revents
) {
527 "Error returned in polling fd %d.\n",
533 "Polling fd %d tells it has hung up.\n",
539 "Polling fd %d tells fd is not open.\n",
544 pthread_rwlock_rdlock(&instance
->fd_pairs_lock
);
545 if(pthread_mutex_trylock(&instance
->fd_pairs
.pair
[i
-inotify_fds
].mutex
) == 0) {
547 "Urgent read on fd %d\n",
549 /* Take care of high priority channels first. */
551 /* it's ok to have an unavailable sub-buffer */
552 ret
= read_subbuffer(instance
, &instance
->fd_pairs
.pair
[i
-inotify_fds
]);
553 if(ret
== EAGAIN
) ret
= 0;
555 ret
= pthread_mutex_unlock(&instance
->fd_pairs
.pair
[i
-inotify_fds
].mutex
);
557 printf("Error in mutex unlock : %s\n", strerror(ret
));
559 pthread_rwlock_unlock(&instance
->fd_pairs_lock
);
563 /* If every buffer FD has hung up, we end the read loop here */
564 if(num_hup
== num_pollfd
- inotify_fds
) break;
567 for(i
=inotify_fds
;i
<num_pollfd
;i
++) {
568 switch(pollfd
[i
].revents
) {
570 pthread_rwlock_rdlock(&instance
->fd_pairs_lock
);
571 if(pthread_mutex_trylock(&instance
->fd_pairs
.pair
[i
-inotify_fds
].mutex
) == 0) {
572 /* Take care of low priority channels. */
574 "Normal read on fd %d\n",
576 /* it's ok to have an unavailable subbuffer */
577 ret
= read_subbuffer(instance
, &instance
->fd_pairs
.pair
[i
-inotify_fds
]);
578 if(ret
== EAGAIN
) ret
= 0;
580 ret
= pthread_mutex_unlock(&instance
->fd_pairs
.pair
[i
-inotify_fds
].mutex
);
582 printf("Error in mutex unlock : %s\n", strerror(ret
));
584 pthread_rwlock_unlock(&instance
->fd_pairs_lock
);
590 /* Update pollfd array if an entry was added to fd_pairs */
591 pthread_rwlock_rdlock(&instance
->fd_pairs_lock
);
592 if((inotify_fds
+ instance
->fd_pairs
.num_pairs
) != num_pollfd
) {
593 pollfd
= realloc(pollfd
,
594 (inotify_fds
+ instance
->fd_pairs
.num_pairs
) * sizeof(struct pollfd
));
595 for(i
=num_pollfd
-inotify_fds
;i
<instance
->fd_pairs
.num_pairs
;i
++) {
596 pollfd
[inotify_fds
+i
].fd
= instance
->fd_pairs
.pair
[i
].channel
;
597 pollfd
[inotify_fds
+i
].events
= POLLIN
|POLLPRI
;
599 num_pollfd
= instance
->fd_pairs
.num_pairs
+ inotify_fds
;
601 pthread_rwlock_unlock(&instance
->fd_pairs_lock
);
603 /* NB: If the fd_pairs structure is updated by another thread from this
604 * point forward, the current thread will wait in the poll without
605 * monitoring the new channel. However, this thread will add the
606 * new channel on next poll (and this should not take too much time
607 * on a loaded system).
609 * This event is quite unlikely and can only occur if a CPU is
610 * hot-plugged while multple lttd threads are running.
622 void close_channel_trace_pairs(struct liblttd_instance
*instance
)
627 for(i
=0;i
<instance
->fd_pairs
.num_pairs
;i
++) {
628 ret
= close(instance
->fd_pairs
.pair
[i
].channel
);
629 if(ret
== -1) perror("Close error on channel");
630 if(instance
->callbacks
->on_close_channel
) {
631 ret
= instance
->callbacks
->on_close_channel(
632 instance
->callbacks
, &instance
->fd_pairs
.pair
[i
]);
633 if(ret
!= 0) perror("Error on close channel callback");
636 free(instance
->fd_pairs
.pair
);
637 free(instance
->inotify_watch_array
.elem
);
641 void * thread_main(void *arg
)
644 struct liblttd_thread_data
*thread_data
= (struct liblttd_thread_data
*) arg
;
646 if(thread_data
->instance
->callbacks
->on_new_thread
)
647 ret
= thread_data
->instance
->callbacks
->on_new_thread(
648 thread_data
->instance
->callbacks
, thread_data
->thread_num
);
653 ret
= read_channels(thread_data
->instance
, thread_data
->thread_num
);
655 if(thread_data
->instance
->callbacks
->on_close_thread
)
656 thread_data
->instance
->callbacks
->on_close_thread(
657 thread_data
->instance
->callbacks
, thread_data
->thread_num
);
664 int channels_init(struct liblttd_instance
*instance
)
668 instance
->inotify_fd
= inotify_init();
669 fcntl(instance
->inotify_fd
, F_SETFL
, O_NONBLOCK
);
671 if(ret
= open_channel_trace_pairs(instance
, instance
->channel_name
,
672 instance
->channel_name
+
673 strlen(instance
->channel_name
)))
675 if (instance
->fd_pairs
.num_pairs
== 0) {
676 printf("No channel available for reading, exiting\n");
681 if(ret
= map_channels(instance
, 0, instance
->fd_pairs
.num_pairs
))
686 close_channel_trace_pairs(instance
);
687 if(instance
->inotify_fd
>= 0)
688 close(instance
->inotify_fd
);
692 int delete_instance(struct liblttd_instance
*instance
)
694 pthread_rwlock_destroy(&instance
->fd_pairs_lock
);
699 int liblttd_start_instance(struct liblttd_instance
*instance
)
709 if(ret
= channels_init(instance
))
712 tids
= malloc(sizeof(pthread_t
) * instance
->num_threads
);
713 for(i
=0; i
<instance
->num_threads
; i
++) {
714 struct liblttd_thread_data
*thread_data
=
715 malloc(sizeof(struct liblttd_thread_data
));
716 thread_data
->thread_num
= i
;
717 thread_data
->instance
= instance
;
719 ret
= pthread_create(&tids
[i
], NULL
, thread_main
, thread_data
);
721 perror("Error creating thread");
726 for(i
=0; i
<instance
->num_threads
; i
++) {
727 ret
= pthread_join(tids
[i
], &tret
);
729 perror("Error joining thread");
732 if((long)tret
!= 0) {
733 printf("Error %s occured in thread %ld\n",
734 strerror((long)tret
), i
);
739 ret
= unmap_channels(instance
);
740 close_channel_trace_pairs(instance
);
741 if(instance
->inotify_fd
>= 0)
742 close(instance
->inotify_fd
);
744 if(instance
->callbacks
->on_trace_end
)
745 instance
->callbacks
->on_trace_end(instance
->callbacks
);
747 delete_instance(instance
);
752 struct liblttd_instance
* liblttd_new_instance(
753 struct liblttd_callbacks
*callbacks
, char *channel_path
,
754 unsigned long n_threads
, int flight_only
, int normal_only
, int verbose
)
756 struct liblttd_instance
* instance
;
757 if(!channel_path
|| !callbacks
) return NULL
;
758 if(n_threads
== 0) n_threads
= 1;
759 if(flight_only
&& normal_only
) return NULL
;
761 instance
= malloc(sizeof(struct liblttd_instance
));
762 if(!instance
) return NULL
;
764 instance
->callbacks
= callbacks
;
766 instance
->inotify_fd
= -1;
768 instance
->fd_pairs
.pair
= NULL
;
769 instance
->fd_pairs
.num_pairs
= 0;
771 instance
->inotify_watch_array
.elem
= NULL
;
772 instance
->inotify_watch_array
.num
= 0;
774 pthread_rwlock_init(&instance
->fd_pairs_lock
, NULL
);
776 strncpy(instance
->channel_name
, channel_path
, PATH_MAX
-1);
777 instance
->num_threads
= n_threads
;
778 instance
->dump_flight_only
= flight_only
;
779 instance
->dump_normal_only
= normal_only
;
780 instance
->verbose_mode
= verbose
;
785 int liblttd_stop_instance(struct liblttd_instance
*instance
)
787 instance
->quit_program
= 1;