3 * Linux Trace Toolkit Daemon
5 * This is a simple daemon that reads a few relay+debugfs channels and save
8 * CPU hot-plugging is supported using inotify.
11 * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
24 #include <sys/types.h>
34 #include <sys/syscall.h>
36 #include <asm/ioctls.h>
38 #include <linux/version.h>
41 #include <asm/ioctl.h>
42 #include <asm/types.h>
44 /* Get the next sub buffer that can be read. */
45 #define RELAY_GET_SUBBUF _IOR(0xF5, 0x00,__u32)
46 /* Release the oldest reserved (by "get") sub buffer. */
47 #define RELAY_PUT_SUBBUF _IOW(0xF5, 0x01,__u32)
48 /* returns the number of sub buffers in the per cpu channel. */
49 #define RELAY_GET_N_SUBBUFS _IOR(0xF5, 0x02,__u32)
50 /* returns the size of the sub buffers. */
51 #define RELAY_GET_SUBBUF_SIZE _IOR(0xF5, 0x03,__u32)
53 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14)
54 #include <sys/inotify.h>
55 #if 0 /* should now be provided by libc. */
56 /* From the inotify-tools 2.6 package */
57 static inline int inotify_init (void)
59 return syscall (__NR_inotify_init
);
62 static inline int inotify_add_watch (int fd
, const char *name
, __u32 mask
)
64 return syscall (__NR_inotify_add_watch
, fd
, name
, mask
);
67 static inline int inotify_rm_watch (int fd
, __u32 wd
)
69 return syscall (__NR_inotify_rm_watch
, fd
, wd
);
74 static inline int inotify_init (void)
79 static inline int inotify_add_watch (int fd
, const char *name
, __u32 mask
)
84 static inline int inotify_rm_watch (int fd
, __u32 wd
)
101 unsigned int n_subbufs
;
102 unsigned int subbuf_size
;
104 pthread_mutex_t mutex
;
107 struct channel_trace_fd
{
108 struct fd_pair
*pair
;
112 struct inotify_watch
{
114 char path_channel
[PATH_MAX
];
115 char path_trace
[PATH_MAX
];
118 struct inotify_watch_array
{
119 struct inotify_watch
*elem
;
123 static __thread
int thread_pipe
[2];
125 struct channel_trace_fd fd_pairs
= { NULL
, 0 };
127 struct inotify_watch_array inotify_watch_array
= { NULL
, 0 };
129 /* protects fd_pairs and inotify_watch_array */
130 pthread_rwlock_t fd_pairs_lock
= PTHREAD_RWLOCK_INITIALIZER
;
133 static char *trace_name
= NULL
;
134 static char *channel_name
= NULL
;
135 static int daemon_mode
= 0;
136 static int append_mode
= 0;
137 static unsigned long num_threads
= 1;
138 volatile static int quit_program
= 0; /* For signal handler */
139 static int dump_flight_only
= 0;
140 static int dump_normal_only
= 0;
141 static int verbose_mode
= 0;
143 #define printf_verbose(fmt, args...) \
146 printf(fmt, ##args); \
151 * -t directory Directory name of the trace to write to. Will be created.
152 * -c directory Root directory of the debugfs trace channels.
153 * -d Run in background (daemon).
154 * -a Trace append mode.
155 * -s Send SIGUSR1 to parent when ready for IO.
157 void show_arguments(void)
159 printf("Please use the following arguments :\n");
161 printf("-t directory Directory name of the trace to write to.\n"
162 " It will be created.\n");
163 printf("-c directory Root directory of the debugfs trace channels.\n");
164 printf("-d Run in background (daemon).\n");
165 printf("-a Append to an possibly existing trace.\n");
166 printf("-N Number of threads to start.\n");
167 printf("-f Dump only flight recorder channels.\n");
168 printf("-n Dump only normal channels.\n");
169 printf("-v Verbose mode.\n");
176 * Parses the command line arguments.
178 * Returns 1 if the arguments were correct, but doesn't ask for program
179 * continuation. Returns -1 if the arguments are incorrect, or 0 if OK.
181 int parse_arguments(int argc
, char **argv
)
187 if(strcmp(argv
[1], "-h") == 0) {
194 switch(argv
[argn
][0]) {
196 switch(argv
[argn
][1]) {
199 trace_name
= argv
[argn
+1];
205 channel_name
= argv
[argn
+1];
217 num_threads
= strtoul(argv
[argn
+1], NULL
, 0);
222 dump_flight_only
= 1;
225 dump_normal_only
= 1;
231 printf("Invalid argument '%s'.\n", argv
[argn
]);
237 printf("Invalid argument '%s'.\n", argv
[argn
]);
244 if(trace_name
== NULL
) {
245 printf("Please specify a trace name.\n");
250 if(channel_name
== NULL
) {
251 printf("Please specify a channel name.\n");
261 printf("Linux Trace Toolkit Trace Daemon " VERSION
"\n");
263 printf("Reading from debugfs directory : %s\n", channel_name
);
264 printf("Writing to trace directory : %s\n", trace_name
);
269 /* signal handling */
271 static void handler(int signo
)
273 printf("Signal %d received : exiting cleanly\n", signo
);
278 int open_buffer_file(char *filename
, char *path_channel
, char *path_trace
,
279 struct channel_trace_fd
*fd_pairs
)
283 struct stat stat_buf
;
285 if(strncmp(filename
, "flight-", sizeof("flight-")-1) != 0) {
286 if(dump_flight_only
) {
287 printf_verbose("Skipping normal channel %s\n",
292 if(dump_normal_only
) {
293 printf_verbose("Skipping flight channel %s\n",
298 printf_verbose("Opening file.\n");
300 fd_pairs
->pair
= realloc(fd_pairs
->pair
,
301 ++fd_pairs
->num_pairs
* sizeof(struct fd_pair
));
303 /* Open the channel in read mode */
304 fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
=
305 open(path_channel
, O_RDONLY
| O_NONBLOCK
);
306 if(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
== -1) {
307 perror(path_channel
);
308 fd_pairs
->num_pairs
--;
309 return 0; /* continue */
311 /* Open the trace in write mode, only append if append_mode */
312 ret
= stat(path_trace
, &stat_buf
);
315 printf_verbose("Appending to file %s as requested\n",
318 fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
=
319 open(path_trace
, O_WRONLY
,
320 S_IRWXU
|S_IRWXG
|S_IRWXO
);
321 if(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
== -1) {
324 close(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
);
325 fd_pairs
->num_pairs
--;
328 ret
= lseek(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
,
333 close(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
);
334 close(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
);
335 fd_pairs
->num_pairs
--;
339 printf("File %s exists, cannot open. Try append mode.\n", path_trace
);
341 close(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
);
342 fd_pairs
->num_pairs
--;
346 if(errno
== ENOENT
) {
347 fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
=
348 open(path_trace
, O_WRONLY
|O_CREAT
|O_EXCL
,
349 S_IRWXU
|S_IRWXG
|S_IRWXO
);
350 if(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].trace
== -1) {
353 close(fd_pairs
->pair
[fd_pairs
->num_pairs
-1].channel
);
354 fd_pairs
->num_pairs
--;
363 int open_channel_trace_pairs(char *subchannel_name
, char *subtrace_name
,
364 struct channel_trace_fd
*fd_pairs
, int *inotify_fd
,
365 struct inotify_watch_array
*iwatch_array
)
367 DIR *channel_dir
= opendir(subchannel_name
);
368 struct dirent
*entry
;
369 struct stat stat_buf
;
371 char path_channel
[PATH_MAX
];
372 int path_channel_len
;
373 char *path_channel_ptr
;
374 char path_trace
[PATH_MAX
];
376 char *path_trace_ptr
;
379 if(channel_dir
== NULL
) {
380 perror(subchannel_name
);
385 printf_verbose("Creating trace subdirectory %s\n", subtrace_name
);
386 ret
= mkdir(subtrace_name
, S_IRWXU
|S_IRWXG
|S_IRWXO
);
388 if(errno
!= EEXIST
) {
389 perror(subtrace_name
);
395 strncpy(path_channel
, subchannel_name
, PATH_MAX
-1);
396 path_channel_len
= strlen(path_channel
);
397 path_channel
[path_channel_len
] = '/';
399 path_channel_ptr
= path_channel
+ path_channel_len
;
401 strncpy(path_trace
, subtrace_name
, PATH_MAX
-1);
402 path_trace_len
= strlen(path_trace
);
403 path_trace
[path_trace_len
] = '/';
405 path_trace_ptr
= path_trace
+ path_trace_len
;
408 iwatch_array
->elem
= realloc(iwatch_array
->elem
,
409 ++iwatch_array
->num
* sizeof(struct inotify_watch
));
411 printf_verbose("Adding inotify for channel %s\n", path_channel
);
412 iwatch_array
->elem
[iwatch_array
->num
-1].wd
= inotify_add_watch(*inotify_fd
, path_channel
, IN_CREATE
);
413 strcpy(iwatch_array
->elem
[iwatch_array
->num
-1].path_channel
, path_channel
);
414 strcpy(iwatch_array
->elem
[iwatch_array
->num
-1].path_trace
, path_trace
);
415 printf_verbose("Added inotify for channel %s, wd %u\n",
416 iwatch_array
->elem
[iwatch_array
->num
-1].path_channel
,
417 iwatch_array
->elem
[iwatch_array
->num
-1].wd
);
420 while((entry
= readdir(channel_dir
)) != NULL
) {
422 if(entry
->d_name
[0] == '.') continue;
424 strncpy(path_channel_ptr
, entry
->d_name
, PATH_MAX
- path_channel_len
);
425 strncpy(path_trace_ptr
, entry
->d_name
, PATH_MAX
- path_trace_len
);
427 ret
= stat(path_channel
, &stat_buf
);
429 perror(path_channel
);
433 printf_verbose("Channel file : %s\n", path_channel
);
435 if(S_ISDIR(stat_buf
.st_mode
)) {
437 printf_verbose("Entering channel subdirectory...\n");
438 ret
= open_channel_trace_pairs(path_channel
, path_trace
, fd_pairs
,
439 inotify_fd
, iwatch_array
);
440 if(ret
< 0) continue;
441 } else if(S_ISREG(stat_buf
.st_mode
)) {
442 open_ret
= open_buffer_file(entry
->d_name
, path_channel
, path_trace
,
450 closedir(channel_dir
);
456 int read_subbuffer(struct fd_pair
*pair
)
458 unsigned int consumed_old
;
465 err
= ioctl(pair
->channel
, RELAY_GET_SUBBUF
, &consumed_old
);
466 printf_verbose("cookie : %u\n", consumed_old
);
469 perror("Reserving sub buffer failed (everything is normal, it is due to concurrency)");
473 err
= TEMP_FAILURE_RETRY(write(pair
->trace
,
475 + (consumed_old
& ((pair
->n_subbufs
* pair
->subbuf_size
)-1)),
480 perror("Error in writing to file");
484 len
= pair
->subbuf_size
;
487 printf_verbose("splice chan to pipe offset %lu\n",
488 (unsigned long)offset
);
489 ret
= splice(pair
->channel
, &offset
, thread_pipe
[1], NULL
,
490 len
, SPLICE_F_MOVE
| SPLICE_F_MORE
);
491 printf_verbose("splice chan to pipe ret %ld\n", ret
);
493 perror("Error in relay splice");
496 ret
= splice(thread_pipe
[0], NULL
, pair
->trace
, NULL
,
497 ret
, SPLICE_F_MOVE
| SPLICE_F_MORE
);
498 printf_verbose("splice pipe to file %ld\n", ret
);
500 perror("Error in file splice");
507 err
= fsync(pair
->trace
);
510 perror("Error in writing to file");
516 err
= ioctl(pair
->channel
, RELAY_PUT_SUBBUF
, &consumed_old
);
519 if(errno
== EFAULT
) {
520 perror("Error in unreserving sub buffer\n");
521 } else if(errno
== EIO
) {
522 perror("Reader has been pushed by the writer, last subbuffer corrupted.");
523 /* FIXME : we may delete the last written buffer if we wish. */
533 int map_channels(struct channel_trace_fd
*fd_pairs
,
534 int idx_begin
, int idx_end
)
539 if(fd_pairs
->num_pairs
<= 0) {
540 printf("No channel to read\n");
544 /* Get the subbuf sizes and number */
546 for(i
=idx_begin
;i
<idx_end
;i
++) {
547 struct fd_pair
*pair
= &fd_pairs
->pair
[i
];
549 ret
= ioctl(pair
->channel
, RELAY_GET_N_SUBBUFS
,
552 perror("Error in getting the number of subbuffers");
555 ret
= ioctl(pair
->channel
, RELAY_GET_SUBBUF_SIZE
,
558 perror("Error in getting the size of the subbuffers");
561 ret
= pthread_mutex_init(&pair
->mutex
, NULL
); /* Fast mutex */
563 perror("Error in mutex init");
570 for(i
=idx_begin
;i
<idx_end
;i
++) {
571 struct fd_pair
*pair
= &fd_pairs
->pair
[i
];
573 pair
->mmap
= mmap(0, pair
->subbuf_size
* pair
->n_subbufs
, PROT_READ
,
574 MAP_SHARED
, pair
->channel
, 0);
575 if(pair
->mmap
== MAP_FAILED
) {
576 perror("Mmap error");
581 goto end
; /* success */
584 /* munmap only the successfully mmapped indexes */
587 for(j
=idx_begin
;j
<i
;j
++) {
588 struct fd_pair
*pair
= &fd_pairs
->pair
[j
];
591 err_ret
= munmap(pair
->mmap
, pair
->subbuf_size
* pair
->n_subbufs
);
593 perror("Error in munmap");
603 int unmap_channels(struct channel_trace_fd
*fd_pairs
)
609 for(j
=0;j
<fd_pairs
->num_pairs
;j
++) {
610 struct fd_pair
*pair
= &fd_pairs
->pair
[j
];
614 err_ret
= munmap(pair
->mmap
, pair
->subbuf_size
* pair
->n_subbufs
);
616 perror("Error in munmap");
620 err_ret
= pthread_mutex_destroy(&pair
->mutex
);
622 perror("Error in mutex destroy");
631 /* Inotify event arrived.
633 * Only support add file for now.
636 int read_inotify(int inotify_fd
,
637 struct channel_trace_fd
*fd_pairs
,
638 struct inotify_watch_array
*iwatch_array
)
640 char buf
[sizeof(struct inotify_event
) + PATH_MAX
];
641 char path_channel
[PATH_MAX
];
642 char path_trace
[PATH_MAX
];
644 struct inotify_event
*ievent
;
651 len
= read(inotify_fd
, buf
, sizeof(struct inotify_event
) + PATH_MAX
);
655 return 0; /* another thread got the data before us */
657 printf("Error in read from inotify FD %s.\n", strerror(len
));
660 while(offset
< len
) {
661 ievent
= (struct inotify_event
*)&(buf
[offset
]);
662 for(i
=0; i
<iwatch_array
->num
; i
++) {
663 if(iwatch_array
->elem
[i
].wd
== ievent
->wd
&&
664 ievent
->mask
== IN_CREATE
) {
666 "inotify wd %u event mask : %u for %s%s\n",
667 ievent
->wd
, ievent
->mask
,
668 iwatch_array
->elem
[i
].path_channel
,
670 old_num
= fd_pairs
->num_pairs
;
671 strcpy(path_channel
, iwatch_array
->elem
[i
].path_channel
);
672 strcat(path_channel
, ievent
->name
);
673 strcpy(path_trace
, iwatch_array
->elem
[i
].path_trace
);
674 strcat(path_trace
, ievent
->name
);
675 if(ret
= open_buffer_file(ievent
->name
, path_channel
,
676 path_trace
, fd_pairs
)) {
677 printf("Error opening buffer file\n");
680 if(ret
= map_channels(fd_pairs
, old_num
, fd_pairs
->num_pairs
)) {
681 printf("Error mapping channel\n");
687 offset
+= sizeof(*ievent
) + ievent
->len
;
696 * Read the debugfs channels and write them in the paired tracefiles.
698 * @fd_pairs : paired channels and trace files.
700 * returns 0 on success, -1 on error.
702 * Note that the high priority polled channels are consumed first. We then poll
703 * again to see if these channels are still in priority. Only when no
704 * high priority channel is left, we start reading low priority channels.
706 * Note that a channel is considered high priority when the buffer is almost
710 int read_channels(unsigned long thread_num
, struct channel_trace_fd
*fd_pairs
,
711 int inotify_fd
, struct inotify_watch_array
*iwatch_array
)
713 struct pollfd
*pollfd
= NULL
;
716 int num_rdy
, num_hup
;
720 unsigned int old_num
;
728 pthread_rwlock_rdlock(&fd_pairs_lock
);
730 /* Start polling the FD. Keep one fd for inotify */
731 pollfd
= malloc((inotify_fds
+ fd_pairs
->num_pairs
) * sizeof(struct pollfd
));
734 pollfd
[0].fd
= inotify_fd
;
735 pollfd
[0].events
= POLLIN
|POLLPRI
;
738 for(i
=0;i
<fd_pairs
->num_pairs
;i
++) {
739 pollfd
[inotify_fds
+i
].fd
= fd_pairs
->pair
[i
].channel
;
740 pollfd
[inotify_fds
+i
].events
= POLLIN
|POLLPRI
;
742 num_pollfd
= inotify_fds
+ fd_pairs
->num_pairs
;
745 pthread_rwlock_unlock(&fd_pairs_lock
);
751 printf("Press a key for next poll...\n");
753 read(STDIN_FILENO
, &buf
, 1);
754 printf("Next poll (polling %d fd) :\n", num_pollfd
);
757 /* Have we received a signal ? */
758 if(quit_program
) break;
760 num_rdy
= poll(pollfd
, num_pollfd
, -1);
763 perror("Poll error");
767 printf_verbose("Data received\n");
769 switch(pollfd
[0].revents
) {
772 "Error returned in polling inotify fd %d.\n",
777 "Polling inotify fd %d tells it has hung up.\n",
782 "Polling inotify fd %d tells fd is not open.\n",
788 "Polling inotify fd %d : data ready.\n",
791 pthread_rwlock_wrlock(&fd_pairs_lock
);
792 read_inotify(inotify_fd
, fd_pairs
, iwatch_array
);
793 pthread_rwlock_unlock(&fd_pairs_lock
);
799 for(i
=inotify_fds
;i
<num_pollfd
;i
++) {
800 switch(pollfd
[i
].revents
) {
803 "Error returned in polling fd %d.\n",
809 "Polling fd %d tells it has hung up.\n",
815 "Polling fd %d tells fd is not open.\n",
820 pthread_rwlock_rdlock(&fd_pairs_lock
);
821 if(pthread_mutex_trylock(&fd_pairs
->pair
[i
-inotify_fds
].mutex
) == 0) {
823 "Urgent read on fd %d\n",
825 /* Take care of high priority channels first. */
827 /* it's ok to have an unavailable subbuffer */
828 ret
= read_subbuffer(&fd_pairs
->pair
[i
-inotify_fds
]);
829 if(ret
== EAGAIN
) ret
= 0;
831 ret
= pthread_mutex_unlock(&fd_pairs
->pair
[i
-inotify_fds
].mutex
);
833 printf("Error in mutex unlock : %s\n", strerror(ret
));
835 pthread_rwlock_unlock(&fd_pairs_lock
);
839 /* If every buffer FD has hung up, we end the read loop here */
840 if(num_hup
== num_pollfd
- inotify_fds
) break;
843 for(i
=inotify_fds
;i
<num_pollfd
;i
++) {
844 switch(pollfd
[i
].revents
) {
846 pthread_rwlock_rdlock(&fd_pairs_lock
);
847 if(pthread_mutex_trylock(&fd_pairs
->pair
[i
-inotify_fds
].mutex
) == 0) {
848 /* Take care of low priority channels. */
850 "Normal read on fd %d\n",
852 /* it's ok to have an unavailable subbuffer */
853 ret
= read_subbuffer(&fd_pairs
->pair
[i
-inotify_fds
]);
854 if(ret
== EAGAIN
) ret
= 0;
856 ret
= pthread_mutex_unlock(&fd_pairs
->pair
[i
-inotify_fds
].mutex
);
858 printf("Error in mutex unlock : %s\n", strerror(ret
));
860 pthread_rwlock_unlock(&fd_pairs_lock
);
866 /* Update pollfd array if an entry was added to fd_pairs */
867 pthread_rwlock_rdlock(&fd_pairs_lock
);
868 if((inotify_fds
+ fd_pairs
->num_pairs
) != num_pollfd
) {
869 pollfd
= realloc(pollfd
,
870 (inotify_fds
+ fd_pairs
->num_pairs
) * sizeof(struct pollfd
));
871 for(i
=num_pollfd
-inotify_fds
;i
<fd_pairs
->num_pairs
;i
++) {
872 pollfd
[inotify_fds
+i
].fd
= fd_pairs
->pair
[i
].channel
;
873 pollfd
[inotify_fds
+i
].events
= POLLIN
|POLLPRI
;
875 num_pollfd
= fd_pairs
->num_pairs
+ inotify_fds
;
877 pthread_rwlock_unlock(&fd_pairs_lock
);
879 /* NB: If the fd_pairs structure is updated by another thread from this
880 * point forward, the current thread will wait in the poll without
881 * monitoring the new channel. However, this thread will add the
882 * new channel on next poll (and this should not take too much time
883 * on a loaded system).
885 * This event is quite unlikely and can only occur if a CPU is
886 * hot-plugged while multple lttd threads are running.
898 void close_channel_trace_pairs(struct channel_trace_fd
*fd_pairs
, int inotify_fd
,
899 struct inotify_watch_array
*iwatch_array
)
904 for(i
=0;i
<fd_pairs
->num_pairs
;i
++) {
905 ret
= close(fd_pairs
->pair
[i
].channel
);
906 if(ret
== -1) perror("Close error on channel");
907 ret
= close(fd_pairs
->pair
[i
].trace
);
908 if(ret
== -1) perror("Close error on trace");
910 free(fd_pairs
->pair
);
911 free(iwatch_array
->elem
);
915 void * thread_main(void *arg
)
918 unsigned long thread_num
= (unsigned long)arg
;
920 ret
= pipe(thread_pipe
);
922 perror("Error creating pipe");
925 ret
= read_channels(thread_num
, &fd_pairs
, inotify_fd
, &inotify_watch_array
);
926 close(thread_pipe
[0]); /* close read end */
927 close(thread_pipe
[1]); /* close write end */
936 inotify_fd
= inotify_init();
937 fcntl(inotify_fd
, F_SETFL
, O_NONBLOCK
);
939 if(ret
= open_channel_trace_pairs(channel_name
, trace_name
, &fd_pairs
,
940 &inotify_fd
, &inotify_watch_array
))
942 if (fd_pairs
.num_pairs
== 0) {
943 printf("No channel available for reading, exiting\n");
947 if(ret
= map_channels(&fd_pairs
, 0, fd_pairs
.num_pairs
))
952 close_channel_trace_pairs(&fd_pairs
, inotify_fd
, &inotify_watch_array
);
959 int main(int argc
, char ** argv
)
962 struct sigaction act
;
967 ret
= parse_arguments(argc
, argv
);
969 if(ret
!= 0) show_arguments();
970 if(ret
< 0) return EINVAL
;
971 if(ret
> 0) return 0;
975 /* Connect the signal handlers */
976 act
.sa_handler
= handler
;
978 sigemptyset(&(act
.sa_mask
));
979 sigaddset(&(act
.sa_mask
), SIGTERM
);
980 sigaddset(&(act
.sa_mask
), SIGQUIT
);
981 sigaddset(&(act
.sa_mask
), SIGINT
);
982 sigaction(SIGTERM
, &act
, NULL
);
983 sigaction(SIGQUIT
, &act
, NULL
);
984 sigaction(SIGINT
, &act
, NULL
);
986 if(ret
= channels_init())
993 perror("An error occured while daemonizing.");
998 tids
= malloc(sizeof(pthread_t
) * num_threads
);
999 for(i
=0; i
<num_threads
; i
++) {
1001 ret
= pthread_create(&tids
[i
], NULL
, thread_main
, (void*)i
);
1003 perror("Error creating thread");
1008 for(i
=0; i
<num_threads
; i
++) {
1009 ret
= pthread_join(tids
[i
], &tret
);
1011 perror("Error joining thread");
1014 if((long)tret
!= 0) {
1015 printf("Error %s occured in thread %u\n",
1016 strerror((long)tret
), i
);
1021 ret
= unmap_channels(&fd_pairs
);
1022 close_channel_trace_pairs(&fd_pairs
, inotify_fd
, &inotify_watch_array
);