src/bin/lttng-sessiond/rotation-thread.c

   1 /*
   2  * Copyright (C) 2017 - Julien Desfossez <jdesfossez@efficios.com>
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License, version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 51
  15  * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16  */
  17
  18 #define _LGPL_SOURCE
  19 #include <lttng/trigger/trigger.h>
  20 #include <common/error.h>
  21 #include <common/config/session-config.h>
  22 #include <common/defaults.h>
  23 #include <common/utils.h>
  24 #include <common/futex.h>
  25 #include <common/align.h>
  26 #include <common/time.h>
  27 #include <common/hashtable/utils.h>
  28 #include <sys/eventfd.h>
  29 #include <sys/stat.h>
  30 #include <time.h>
  31 #include <signal.h>
  32 #include <inttypes.h>
  33
  34 #include <common/kernel-ctl/kernel-ctl.h>
  35 #include <lttng/notification/channel-internal.h>
  36 #include <lttng/rotate-internal.h>
  37
  38 #include "rotation-thread.h"
  39 #include "lttng-sessiond.h"
  40 #include "health-sessiond.h"
  41 #include "rotate.h"
  42 #include "cmd.h"
  43 #include "session.h"
  44 #include "sessiond-timer.h"
  45
  46 #include <urcu.h>
  47 #include <urcu/list.h>
  48 #include <urcu/rculfhash.h>
  49
  50 /*
  51  * Store a struct rotation_channel_info for each channel that is currently
  52  * being rotated by the consumer.
  53  */
  54 struct cds_lfht *channel_pending_rotate_ht;
  55
  56 struct rotation_thread_state {
  57         struct lttng_poll_event events;
  58 };
  59
  60 static
  61 void channel_rotation_info_destroy(struct rotation_channel_info *channel_info)
  62 {
  63         assert(channel_info);
  64         free(channel_info);
  65 }
  66
  67 static
  68 int match_channel_info(struct cds_lfht_node *node, const void *key)
  69 {
  70         struct rotation_channel_key *channel_key = (struct rotation_channel_key *) key;
  71         struct rotation_channel_info *channel_info;
  72
  73         channel_info = caa_container_of(node, struct rotation_channel_info,
  74                         rotate_channels_ht_node);
  75
  76         return !!((channel_key->key == channel_info->channel_key.key) &&
  77                         (channel_key->domain == channel_info->channel_key.domain));
  78 }
  79
  80 static
  81 struct rotation_channel_info *lookup_channel_pending(uint64_t key,
  82                 enum lttng_domain_type domain)
  83 {
  84         struct cds_lfht_iter iter;
  85         struct cds_lfht_node *node;
  86         struct rotation_channel_info *channel_info = NULL;
  87         struct rotation_channel_key channel_key = { .key = key,
  88                         .domain = domain };
  89
  90         cds_lfht_lookup(channel_pending_rotate_ht,
  91                         hash_channel_key(&channel_key),
  92                         match_channel_info,
  93                         &channel_key, &iter);
  94         node = cds_lfht_iter_get_node(&iter);
  95         if (!node) {
  96                 goto end;
  97         }
  98
  99         channel_info = caa_container_of(node, struct rotation_channel_info,
 100                         rotate_channels_ht_node);
 101         cds_lfht_del(channel_pending_rotate_ht, node);
 102 end:
 103         return channel_info;
 104 }
 105
 106 /*
 107  * Destroy the thread data previously created by the init function.
 108  */
 109 void rotation_thread_handle_destroy(
 110                 struct rotation_thread_handle *handle)
 111 {
 112         int ret;
 113
 114         if (!handle) {
 115                 goto end;
 116         }
 117
 118         if (handle->ust32_consumer >= 0) {
 119                 ret = close(handle->ust32_consumer);
 120                 if (ret) {
 121                         PERROR("close 32-bit consumer channel rotation pipe");
 122                 }
 123         }
 124         if (handle->ust64_consumer >= 0) {
 125                 ret = close(handle->ust64_consumer);
 126                 if (ret) {
 127                         PERROR("close 64-bit consumer channel rotation pipe");
 128                 }
 129         }
 130         if (handle->kernel_consumer >= 0) {
 131                 ret = close(handle->kernel_consumer);
 132                 if (ret) {
 133                         PERROR("close kernel consumer channel rotation pipe");
 134                 }
 135         }
 136
 137 end:
 138         free(handle);
 139 }
 140
 141 struct rotation_thread_handle *rotation_thread_handle_create(
 142                 struct lttng_pipe *ust32_channel_rotate_pipe,
 143                 struct lttng_pipe *ust64_channel_rotate_pipe,
 144                 struct lttng_pipe *kernel_channel_rotate_pipe,
 145                 int thread_quit_pipe,
 146                 struct rotation_thread_timer_queue *rotation_timer_queue)
 147 {
 148         struct rotation_thread_handle *handle;
 149
 150         handle = zmalloc(sizeof(*handle));
 151         if (!handle) {
 152                 goto end;
 153         }
 154
 155         if (ust32_channel_rotate_pipe) {
 156                 handle->ust32_consumer =
 157                                 lttng_pipe_release_readfd(
 158                                         ust32_channel_rotate_pipe);
 159                 if (handle->ust32_consumer < 0) {
 160                         goto error;
 161                 }
 162         } else {
 163                 handle->ust32_consumer = -1;
 164         }
 165         if (ust64_channel_rotate_pipe) {
 166                 handle->ust64_consumer =
 167                                 lttng_pipe_release_readfd(
 168                                         ust64_channel_rotate_pipe);
 169                 if (handle->ust64_consumer < 0) {
 170                         goto error;
 171                 }
 172         } else {
 173                 handle->ust64_consumer = -1;
 174         }
 175         if (kernel_channel_rotate_pipe) {
 176                 handle->kernel_consumer =
 177                                 lttng_pipe_release_readfd(
 178                                         kernel_channel_rotate_pipe);
 179                 if (handle->kernel_consumer < 0) {
 180                         goto error;
 181                 }
 182         } else {
 183                 handle->kernel_consumer = -1;
 184         }
 185         handle->thread_quit_pipe = thread_quit_pipe;
 186         handle->rotation_timer_queue = rotation_timer_queue;
 187
 188 end:
 189         return handle;
 190 error:
 191         rotation_thread_handle_destroy(handle);
 192         return NULL;
 193 }
 194
 195 static
 196 int init_poll_set(struct lttng_poll_event *poll_set,
 197                 struct rotation_thread_handle *handle)
 198 {
 199         int ret;
 200
 201         /*
 202          * Create pollset with size 5:
 203          *      - sessiond quit pipe
 204          *      - sessiond timer pipe,
 205          *      - consumerd (32-bit user space) channel rotate pipe,
 206          *      - consumerd (64-bit user space) channel rotate pipe,
 207          *      - consumerd (kernel) channel rotate pipe,
 208          */
 209         ret = lttng_poll_create(poll_set, 5, LTTNG_CLOEXEC);
 210         if (ret < 0) {
 211                 goto end;
 212         }
 213
 214         ret = lttng_poll_add(poll_set, handle->thread_quit_pipe,
 215                         LPOLLIN | LPOLLERR);
 216         if (ret < 0) {
 217                 ERR("[rotation-thread] Failed to add thread_quit_pipe fd to pollset");
 218                 goto error;
 219         }
 220         ret = lttng_poll_add(poll_set,
 221                         lttng_pipe_get_readfd(handle->rotation_timer_queue->event_pipe),
 222                         LPOLLIN | LPOLLERR);
 223         if (ret < 0) {
 224                 ERR("[rotation-thread] Failed to add rotate_pending fd to pollset");
 225                 goto error;
 226         }
 227         ret = lttng_poll_add(poll_set, handle->ust32_consumer,
 228                         LPOLLIN | LPOLLERR);
 229         if (ret < 0) {
 230                 ERR("[rotation-thread] Failed to add ust-32 channel rotation pipe fd to pollset");
 231                 goto error;
 232         }
 233         ret = lttng_poll_add(poll_set, handle->ust64_consumer,
 234                         LPOLLIN | LPOLLERR);
 235         if (ret < 0) {
 236                 ERR("[rotation-thread] Failed to add ust-64 channel rotation pipe fd to pollset");
 237                 goto error;
 238         }
 239         if (handle->kernel_consumer >= 0) {
 240                 ret = lttng_poll_add(poll_set, handle->kernel_consumer,
 241                                 LPOLLIN | LPOLLERR);
 242                 if (ret < 0) {
 243                         ERR("[rotation-thread] Failed to add kernel channel rotation pipe fd to pollset");
 244                         goto error;
 245                 }
 246         }
 247
 248 end:
 249         return ret;
 250 error:
 251         lttng_poll_clean(poll_set);
 252         return ret;
 253 }
 254
 255 static
 256 void fini_thread_state(struct rotation_thread_state *state)
 257 {
 258         lttng_poll_clean(&state->events);
 259         cds_lfht_destroy(channel_pending_rotate_ht, NULL);
 260 }
 261
 262 static
 263 int init_thread_state(struct rotation_thread_handle *handle,
 264                 struct rotation_thread_state *state)
 265 {
 266         int ret;
 267
 268         memset(state, 0, sizeof(*state));
 269         lttng_poll_init(&state->events);
 270
 271         ret = init_poll_set(&state->events, handle);
 272         if (ret) {
 273                 ERR("[rotation-thread] Failed to initialize rotation thread poll set");
 274                 goto end;
 275         }
 276
 277         channel_pending_rotate_ht = cds_lfht_new(DEFAULT_HT_SIZE,
 278                         1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, NULL);
 279         if (!channel_pending_rotate_ht) {
 280                 ERR("[rotation-thread] Failed to create channel pending rotation hash table");
 281                 ret = -1;
 282                 goto end;
 283         }
 284
 285 end:
 286         return ret;
 287 }
 288
 289 static
 290 int handle_channel_rotation_pipe(int fd, uint32_t revents,
 291                 struct rotation_thread_handle *handle,
 292                 struct rotation_thread_state *state)
 293 {
 294         int ret = 0;
 295         enum lttng_domain_type domain;
 296         struct rotation_channel_info *channel_info;
 297         struct ltt_session *session = NULL;
 298         uint64_t key;
 299
 300         if (fd == handle->ust32_consumer ||
 301                         fd == handle->ust64_consumer) {
 302                 domain = LTTNG_DOMAIN_UST;
 303         } else if (fd == handle->kernel_consumer) {
 304                 domain = LTTNG_DOMAIN_KERNEL;
 305         } else {
 306                 ERR("[rotation-thread] Unknown channel rotation pipe fd %d",
 307                                 fd);
 308                 abort();
 309         }
 310
 311         if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
 312                 ret = lttng_poll_del(&state->events, fd);
 313                 if (ret) {
 314                         ERR("[rotation-thread] Failed to remove consumer "
 315                                         "rotation pipe from poll set");
 316                 }
 317                 goto end;
 318         }
 319
 320         do {
 321                 ret = read(fd, &key, sizeof(key));
 322         } while (ret == -1 && errno == EINTR);
 323         if (ret != sizeof(key)) {
 324                 ERR("[rotation-thread] Failed to read from pipe (fd = %i)",
 325                                 fd);
 326                 ret = -1;
 327                 goto end;
 328         }
 329
 330         DBG("[rotation-thread] Received notification for chan %" PRIu64
 331                         ", domain %d\n", key, domain);
 332
 333         channel_info = lookup_channel_pending(key, domain);
 334         if (!channel_info) {
 335                 ERR("[rotation-thread] Failed to find channel_info (key = %"
 336                                 PRIu64 ")", key);
 337                 ret = -1;
 338                 goto end;
 339         }
 340         rcu_read_lock();
 341         session_lock_list();
 342         session = session_find_by_id(channel_info->session_id);
 343         if (!session) {
 344                 /*
 345                  * The session may have been destroyed before we had a chance to
 346                  * perform this action, return gracefully.
 347                  */
 348                 DBG("[rotation-thread] Session %" PRIu64 " not found",
 349                                 channel_info->session_id);
 350                 ret = 0;
 351                 goto end_unlock_session_list;
 352         }
 353
 354         session_lock(session);
 355         if (--session->nr_chan_rotate_pending == 0) {
 356                 time_t now = time(NULL);
 357
 358                 if (now == (time_t) -1) {
 359                         session->rotation_state = LTTNG_ROTATION_STATE_ERROR;
 360                         ret = LTTNG_ERR_UNK;
 361                         goto end_unlock_session;
 362                 }
 363
 364                 ret = rename_complete_chunk(session, now);
 365                 if (ret < 0) {
 366                         ERR("Failed to rename completed rotation chunk");
 367                         goto end_unlock_session;
 368                 }
 369                 session->rotate_pending = false;
 370                 session->rotation_state = LTTNG_ROTATION_STATE_COMPLETED;
 371                 session->last_chunk_start_ts = session->current_chunk_start_ts;
 372                 if (session->rotate_pending_relay) {
 373                         ret = sessiond_timer_rotate_pending_start(
 374                                         session,
 375                                         DEFAULT_ROTATE_PENDING_RELAY_TIMER);
 376                         if (ret) {
 377                                 ERR("Failed to enable rotate pending timer");
 378                                 ret = -1;
 379                                 goto end_unlock_session;
 380                         }
 381                 }
 382                 DBG("Rotation completed for session %s", session->name);
 383         }
 384
 385         ret = 0;
 386
 387 end_unlock_session:
 388         channel_rotation_info_destroy(channel_info);
 389         session_unlock(session);
 390 end_unlock_session_list:
 391         session_unlock_list();
 392         rcu_read_unlock();
 393 end:
 394         return ret;
 395 }
 396
 397 /*
 398  * Process the rotate_pending check, called with session lock held.
 399  */
 400 static
 401 int rotate_pending_relay_timer(struct ltt_session *session)
 402 {
 403         int ret;
 404
 405         DBG("[rotation-thread] Check rotate pending on session %" PRIu64,
 406                         session->id);
 407         ret = relay_rotate_pending(session, session->rotate_count - 1);
 408         if (ret < 0) {
 409                 ERR("[rotation-thread] Check relay rotate pending");
 410                 goto end;
 411         }
 412         if (ret == 0) {
 413                 DBG("[rotation-thread] Rotation completed on the relay for "
 414                                 "session %" PRIu64, session->id);
 415                 /*
 416                  * Now we can clear the pending flag in the session. New
 417                  * rotations can start now.
 418                  */
 419                 session->rotate_pending_relay = false;
 420         } else if (ret == 1) {
 421                 DBG("[rotation-thread] Rotation still pending on the relay for "
 422                                 "session %" PRIu64, session->id);
 423                 ret = sessiond_timer_rotate_pending_start(session,
 424                                 DEFAULT_ROTATE_PENDING_RELAY_TIMER);
 425                 if (ret) {
 426                         ERR("Re-enabling rotate pending timer");
 427                         ret = -1;
 428                         goto end;
 429                 }
 430         }
 431
 432         ret = 0;
 433
 434 end:
 435         return ret;
 436 }
 437
 438 static
 439 int handle_rotate_timer_pipe(uint32_t revents,
 440                 struct rotation_thread_handle *handle,
 441                 struct rotation_thread_state *state,
 442                 struct rotation_thread_timer_queue *queue)
 443 {
 444         int ret = 0;
 445         int fd = lttng_pipe_get_readfd(queue->event_pipe);
 446         struct ltt_session *session;
 447         char buf[1];
 448
 449         if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
 450                 ret = lttng_poll_del(&state->events, fd);
 451                 if (ret) {
 452                         ERR("[rotation-thread] Failed to remove consumer "
 453                                         "rotate pending pipe from poll set");
 454                 }
 455                 goto end;
 456         }
 457
 458         ret = lttng_read(fd, buf, 1);
 459         if (ret != 1) {
 460                 ERR("[rotation-thread] Failed to read from wakeup pipe (fd = %i)", fd);
 461                 ret = -1;
 462                 goto end;
 463         }
 464
 465         for (;;) {
 466                 struct sessiond_rotation_timer *timer_data;
 467
 468                 /*
 469                  * Take the queue lock only to pop elements from the list.
 470                  */
 471                 pthread_mutex_lock(&queue->lock);
 472                 if (cds_list_empty(&queue->list)) {
 473                         pthread_mutex_unlock(&queue->lock);
 474                         break;
 475                 }
 476                 timer_data = cds_list_first_entry(&queue->list,
 477                                 struct sessiond_rotation_timer, head);
 478                 cds_list_del(&timer_data->head);
 479                 pthread_mutex_unlock(&queue->lock);
 480
 481                 /*
 482                  * session lock to lookup the session ID.
 483                  */
 484                 session_lock_list();
 485                 session = session_find_by_id(timer_data->session_id);
 486                 if (!session) {
 487                         DBG("[rotation-thread] Session %" PRIu64 " not found",
 488                                         timer_data->session_id);
 489                         /*
 490                          * This is a non-fatal error, and we cannot report it to the
 491                          * user (timer), so just print the error and continue the
 492                          * processing.
 493                          */
 494                         session_unlock_list();
 495                         free(timer_data);
 496                         continue;
 497                 }
 498
 499                 /*
 500                  * Take the session lock and release the session_list lock.
 501                  */
 502                 session_lock(session);
 503                 session_unlock_list();
 504
 505                 if (timer_data->signal == LTTNG_SESSIOND_SIG_ROTATE_PENDING) {
 506                         ret = rotate_pending_relay_timer(session);
 507                 } else {
 508                         ERR("Unknown signal in rotate timer %d", timer_data->signal);
 509                         ret = -1;
 510                 }
 511                 session_unlock(session);
 512                 free(timer_data);
 513                 if (ret) {
 514                         ERR("Error processing timer");
 515                         goto end;
 516                 }
 517         }
 518
 519         ret = 0;
 520
 521 end:
 522         return ret;
 523 }
 524
 525 void *thread_rotation(void *data)
 526 {
 527         int ret;
 528         struct rotation_thread_handle *handle = data;
 529         struct rotation_thread_state state;
 530
 531         DBG("[rotation-thread] Started rotation thread");
 532
 533         if (!handle) {
 534                 ERR("[rotation-thread] Invalid thread context provided");
 535                 goto end;
 536         }
 537
 538         rcu_register_thread();
 539         rcu_thread_online();
 540
 541         health_register(health_sessiond, HEALTH_SESSIOND_TYPE_ROTATION);
 542         health_code_update();
 543
 544         ret = init_thread_state(handle, &state);
 545         if (ret) {
 546                 goto end;
 547         }
 548
 549         /* Ready to handle client connections. */
 550         sessiond_notify_ready();
 551
 552         while (true) {
 553                 int fd_count, i;
 554
 555                 health_poll_entry();
 556                 DBG("[rotation-thread] Entering poll wait");
 557                 ret = lttng_poll_wait(&state.events, -1);
 558                 DBG("[rotation-thread] Poll wait returned (%i)", ret);
 559                 health_poll_exit();
 560                 if (ret < 0) {
 561                         /*
 562                          * Restart interrupted system call.
 563                          */
 564                         if (errno == EINTR) {
 565                                 continue;
 566                         }
 567                         ERR("[rotation-thread] Error encountered during lttng_poll_wait (%i)", ret);
 568                         goto error;
 569                 }
 570
 571                 fd_count = ret;
 572                 for (i = 0; i < fd_count; i++) {
 573                         int fd = LTTNG_POLL_GETFD(&state.events, i);
 574                         uint32_t revents = LTTNG_POLL_GETEV(&state.events, i);
 575
 576                         DBG("[rotation-thread] Handling fd (%i) activity (%u)",
 577                                         fd, revents);
 578
 579                         if (fd == handle->thread_quit_pipe) {
 580                                 DBG("[rotation-thread] Quit pipe activity");
 581                                 goto exit;
 582                         } else if (fd == lttng_pipe_get_readfd(handle->rotation_timer_queue->event_pipe)) {
 583                                 ret = handle_rotate_timer_pipe(revents,
 584                                                 handle, &state, handle->rotation_timer_queue);
 585                                 if (ret) {
 586                                         ERR("[rotation-thread] Failed to handle rotation timer pipe event");
 587                                         goto error;
 588                                 }
 589                         } else if (fd == handle->ust32_consumer ||
 590                                         fd == handle->ust64_consumer ||
 591                                         fd == handle->kernel_consumer) {
 592                                 ret = handle_channel_rotation_pipe(fd,
 593                                                 revents, handle, &state);
 594                                 if (ret) {
 595                                         ERR("[rotation-thread] Handle channel rotation pipe");
 596                                         goto error;
 597                                 }
 598                         }
 599                 }
 600         }
 601 exit:
 602 error:
 603         DBG("[rotation-thread] Exit");
 604         fini_thread_state(&state);
 605         health_unregister(health_sessiond);
 606         rcu_thread_offline();
 607         rcu_unregister_thread();
 608 end:
 609         return NULL;
 610 }