src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include "condition-internal.hpp"
  10 #include "consumer.hpp"
  11 #include "event-notifier-error-accounting.hpp"
  12 #include "kern-modules.hpp"
  13 #include "kernel-consumer.hpp"
  14 #include "kernel.hpp"
  15 #include "lttng-sessiond.hpp"
  16 #include "lttng-syscall.hpp"
  17 #include "modprobe.hpp"
  18 #include "notification-thread-commands.hpp"
  19 #include "sessiond-config.hpp"
  20 #include "tracker.hpp"
  21 #include "utils.hpp"
  22
  23 #include <common/common.hpp>
  24 #include <common/hashtable/utils.hpp>
  25 #include <common/kernel-ctl/kernel-ctl.hpp>
  26 #include <common/kernel-ctl/kernel-ioctl.hpp>
  27 #include <common/scope-exit.hpp>
  28 #include <common/sessiond-comm/sessiond-comm.hpp>
  29 #include <common/trace-chunk.hpp>
  30 #include <common/tracker.hpp>
  31 #include <common/urcu.hpp>
  32 #include <common/utils.hpp>
  33
  34 #include <lttng/condition/event-rule-matches-internal.hpp>
  35 #include <lttng/condition/event-rule-matches.h>
  36 #include <lttng/event-rule/event-rule-internal.hpp>
  37 #include <lttng/event-rule/event-rule.h>
  38 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  39 #include <lttng/event.h>
  40 #include <lttng/lttng-error.h>
  41 #include <lttng/tracker.h>
  42 #include <lttng/userspace-probe-internal.hpp>
  43 #include <lttng/userspace-probe.h>
  44
  45 #include <fcntl.h>
  46 #include <inttypes.h>
  47 #include <stdio.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <sys/types.h>
  51 #include <unistd.h>
  52
  53 namespace {
  54 /*
  55  * Key used to reference a channel between the sessiond and the consumer. This
  56  * is only read and updated with the session_list lock held.
  57  */
  58 uint64_t next_kernel_channel_key;
  59
  60 const char *module_proc_lttng = "/proc/lttng";
  61
  62 int kernel_tracer_fd = -1;
  63 nonstd::optional<enum lttng_kernel_tracer_status> kernel_tracer_status = nonstd::nullopt;
  64 int kernel_tracer_event_notifier_group_fd = -1;
  65 int kernel_tracer_event_notifier_group_notification_fd = -1;
  66 struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  67
  68 const char *kernel_tracer_status_to_str(lttng_kernel_tracer_status status)
  69 {
  70         switch (status) {
  71         case LTTNG_KERNEL_TRACER_STATUS_INITIALIZED:
  72                 return "LTTNG_KERNEL_TRACER_STATUS_INITIALIZED";
  73         case LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN:
  74                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN";
  75         case LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT:
  76                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT";
  77         case LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER:
  78                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER";
  79         case LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG:
  80                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG";
  81         case LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH:
  82                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH";
  83         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN:
  84                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN";
  85         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING:
  86                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING";
  87         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE:
  88                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE";
  89         }
  90
  91         abort();
  92 }
  93
  94 /*
  95  * On some architectures, calling convention details are embedded in the symbol
  96  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  97  * where an instruction boundary would be legal) to add
  98  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  99  * a per-architecture basis.
 100  */
 101 #if defined(__arm__) || defined(__aarch64__)
 102 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
 103 {
 104         /*
 105          * The least significant bit is used when branching to switch to thumb
 106          * ISA. However, it's an invalid address for us; mask the least
 107          * significant bit.
 108          */
 109         return raw_offset &= ~0b1;
 110 }
 111 #else /* defined(__arm__) || defined(__aarch64__) */
 112 inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
 113 {
 114         return raw_offset;
 115 }
 116 #endif
 117 } /* namespace */
 118
 119 /*
 120  * Add context on a kernel channel.
 121  *
 122  * Assumes the ownership of ctx.
 123  */
 124 int kernel_add_channel_context(struct ltt_kernel_channel *chan, struct ltt_kernel_context *ctx)
 125 {
 126         int ret;
 127
 128         LTTNG_ASSERT(chan);
 129         LTTNG_ASSERT(ctx);
 130
 131         DBG("Adding context to channel %s", chan->channel->name);
 132         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 133         if (ret < 0) {
 134                 switch (-ret) {
 135                 case ENOSYS:
 136                         /* Exists but not available for this kernel */
 137                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 138                         goto error;
 139                 case EEXIST:
 140                         /* If EEXIST, we just ignore the error */
 141                         ret = 0;
 142                         goto end;
 143                 default:
 144                         PERROR("add context ioctl");
 145                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 146                         goto error;
 147                 }
 148         }
 149         ret = 0;
 150
 151 end:
 152         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 153         ctx->in_list = true;
 154         ctx = nullptr;
 155 error:
 156         if (ctx) {
 157                 trace_kernel_destroy_context(ctx);
 158         }
 159         return ret;
 160 }
 161
 162 /*
 163  * Create a new kernel session, register it to the kernel tracer and add it to
 164  * the session daemon session.
 165  */
 166 int kernel_create_session(const ltt_session::locked_ref& session)
 167 {
 168         int ret;
 169         struct ltt_kernel_session *lks;
 170
 171         /* Allocate data structure */
 172         lks = trace_kernel_create_session();
 173         if (lks == nullptr) {
 174                 ret = -1;
 175                 goto error;
 176         }
 177
 178         /* Kernel tracer session creation */
 179         ret = kernctl_create_session(kernel_tracer_fd);
 180         if (ret < 0) {
 181                 PERROR("ioctl kernel create session");
 182                 goto error;
 183         }
 184
 185         lks->fd = ret;
 186         /* Prevent fd duplication after execlp() */
 187         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 188         if (ret < 0) {
 189                 PERROR("fcntl session fd");
 190         }
 191
 192         lks->id = session->id;
 193         lks->consumer_fds_sent = 0;
 194         session->kernel_session = lks;
 195
 196         DBG("Kernel session created (fd: %d)", lks->fd);
 197
 198         /*
 199          * This is necessary since the creation time is present in the session
 200          * name when it is generated.
 201          */
 202         if (session->has_auto_generated_name) {
 203                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 204         } else {
 205                 ret = kernctl_session_set_name(lks->fd, session->name);
 206         }
 207         if (ret) {
 208                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 209                      session->id,
 210                      session->name);
 211         }
 212
 213         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 214         if (ret) {
 215                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 216                      session->id,
 217                      session->name);
 218         }
 219
 220         return 0;
 221
 222 error:
 223         if (lks) {
 224                 trace_kernel_destroy_session(lks);
 225                 trace_kernel_free_session(lks);
 226         }
 227         return ret;
 228 }
 229
 230 /*
 231  * Create a kernel channel, register it to the kernel tracer and add it to the
 232  * kernel session.
 233  */
 234 int kernel_create_channel(struct ltt_kernel_session *session, struct lttng_channel *chan)
 235 {
 236         int ret;
 237         struct ltt_kernel_channel *lkc;
 238
 239         LTTNG_ASSERT(session);
 240         LTTNG_ASSERT(chan);
 241
 242         /* Allocate kernel channel */
 243         lkc = trace_kernel_create_channel(chan);
 244         if (lkc == nullptr) {
 245                 goto error;
 246         }
 247
 248         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 249              chan->name,
 250              lkc->channel->attr.overwrite,
 251              lkc->channel->attr.subbuf_size,
 252              lkc->channel->attr.num_subbuf,
 253              lkc->channel->attr.switch_timer_interval,
 254              lkc->channel->attr.read_timer_interval,
 255              lkc->channel->attr.live_timer_interval,
 256              lkc->channel->attr.output);
 257
 258         /* Kernel tracer channel creation */
 259         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 260         if (ret < 0) {
 261                 PERROR("ioctl kernel create channel");
 262                 goto error;
 263         }
 264
 265         /* Setup the channel fd */
 266         lkc->fd = ret;
 267         /* Prevent fd duplication after execlp() */
 268         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 269         if (ret < 0) {
 270                 PERROR("fcntl session fd");
 271         }
 272
 273         /* Add channel to session */
 274         cds_list_add(&lkc->list, &session->channel_list.head);
 275         session->channel_count++;
 276         lkc->session = session;
 277         lkc->key = ++next_kernel_channel_key;
 278
 279         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 280             lkc->channel->name,
 281             lkc->fd,
 282             lkc->key);
 283
 284         return 0;
 285
 286 error:
 287         if (lkc) {
 288                 free(lkc->channel);
 289                 free(lkc);
 290         }
 291         return -1;
 292 }
 293
 294 /*
 295  * Create a kernel event notifier group, register it to the kernel tracer and
 296  * add it to the kernel session.
 297  */
 298 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 299 {
 300         int ret;
 301         int local_fd = -1;
 302
 303         LTTNG_ASSERT(event_notifier_group_fd);
 304
 305         /* Kernel event notifier group creation. */
 306         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 307         if (ret < 0) {
 308                 PERROR("Failed to create kernel event notifier group");
 309                 ret = -1;
 310                 goto error;
 311         }
 312
 313         local_fd = ret;
 314
 315         /* Prevent fd duplication after execlp(). */
 316         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 317         if (ret < 0) {
 318                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 319                        local_fd);
 320                 goto error;
 321         }
 322
 323         DBG("Created kernel event notifier group: fd = %d", local_fd);
 324         *event_notifier_group_fd = local_fd;
 325         local_fd = -1;
 326         ret = 0;
 327 error:
 328         if (local_fd >= 0) {
 329                 ret = close(local_fd);
 330                 if (ret) {
 331                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 332                                local_fd);
 333                 }
 334         }
 335
 336         return ret;
 337 }
 338
 339 /*
 340  * Compute the offset of the instrumentation byte in the binary based on the
 341  * function probe location using the ELF lookup method.
 342  *
 343  * Returns 0 on success and set the offset out parameter to the offset of the
 344  * elf symbol
 345  * Returns -1 on error
 346  */
 347 static int extract_userspace_probe_offset_function_elf(
 348         const struct lttng_userspace_probe_location *probe_location,
 349         uid_t uid,
 350         gid_t gid,
 351         uint64_t *offset)
 352 {
 353         int fd;
 354         int ret = 0;
 355         const char *symbol = nullptr;
 356         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 357         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 358
 359         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 360                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 361
 362         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 363         if (!lookup) {
 364                 ret = -1;
 365                 goto end;
 366         }
 367
 368         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 369
 370         LTTNG_ASSERT(lookup_method_type ==
 371                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 372
 373         symbol = lttng_userspace_probe_location_function_get_function_name(probe_location);
 374         if (!symbol) {
 375                 ret = -1;
 376                 goto end;
 377         }
 378
 379         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 380         if (fd < 0) {
 381                 ret = -1;
 382                 goto end;
 383         }
 384
 385         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 386         if (ret < 0) {
 387                 DBG("userspace probe offset calculation failed for "
 388                     "function %s",
 389                     symbol);
 390                 goto end;
 391         }
 392
 393         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t) (*offset));
 394 end:
 395         return ret;
 396 }
 397
 398 /*
 399  * Compute the offsets of the instrumentation bytes in the binary based on the
 400  * tracepoint probe location using the SDT lookup method. This function
 401  * allocates the offsets buffer, the caller must free it.
 402  *
 403  * Returns 0 on success and set the offset out parameter to the offsets of the
 404  * SDT tracepoint.
 405  * Returns -1 on error.
 406  */
 407 static int extract_userspace_probe_offset_tracepoint_sdt(
 408         const struct lttng_userspace_probe_location *probe_location,
 409         uid_t uid,
 410         gid_t gid,
 411         uint64_t **offsets,
 412         uint32_t *offsets_count)
 413 {
 414         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 415         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 416         const char *probe_name = nullptr, *provider_name = nullptr;
 417         int ret = 0;
 418         int fd, i;
 419
 420         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 421                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 422
 423         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 424         if (!lookup) {
 425                 ret = -1;
 426                 goto end;
 427         }
 428
 429         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 430
 431         LTTNG_ASSERT(lookup_method_type ==
 432                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 433
 434         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(probe_location);
 435         if (!probe_name) {
 436                 ret = -1;
 437                 goto end;
 438         }
 439
 440         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(probe_location);
 441         if (!provider_name) {
 442                 ret = -1;
 443                 goto end;
 444         }
 445
 446         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 447         if (fd < 0) {
 448                 ret = -1;
 449                 goto end;
 450         }
 451
 452         ret = run_as_extract_sdt_probe_offsets(
 453                 fd, provider_name, probe_name, uid, gid, offsets, offsets_count);
 454         if (ret < 0) {
 455                 DBG("userspace probe offset calculation failed for sdt "
 456                     "probe %s:%s",
 457                     provider_name,
 458                     probe_name);
 459                 goto end;
 460         }
 461
 462         if (*offsets_count == 0) {
 463                 DBG("no userspace probe offset found");
 464                 goto end;
 465         }
 466
 467         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 468             *offsets_count,
 469             provider_name,
 470             probe_name);
 471         for (i = 0; i < *offsets_count; i++) {
 472                 DBG("\t0x%jd", (intmax_t) ((*offsets)[i]));
 473         }
 474 end:
 475         return ret;
 476 }
 477
 478 static int userspace_probe_add_callsite(const struct lttng_userspace_probe_location *location,
 479                                         uid_t uid,
 480                                         gid_t gid,
 481                                         int fd)
 482 {
 483         const struct lttng_userspace_probe_location_lookup_method *lookup_method = nullptr;
 484         enum lttng_userspace_probe_location_lookup_method_type type;
 485         int ret;
 486
 487         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 488         if (!lookup_method) {
 489                 ret = -1;
 490                 goto end;
 491         }
 492
 493         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 494         switch (type) {
 495         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 496         {
 497                 struct lttng_kernel_abi_event_callsite callsite;
 498                 uint64_t offset;
 499
 500                 ret = extract_userspace_probe_offset_function_elf(location, uid, gid, &offset);
 501                 if (ret) {
 502                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 503                         goto end;
 504                 }
 505
 506                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 507                 ret = kernctl_add_callsite(fd, &callsite);
 508                 if (ret) {
 509                         WARN("Failed to add callsite to ELF userspace probe.");
 510                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 511                         goto end;
 512                 }
 513                 break;
 514         }
 515         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 516         {
 517                 int i;
 518                 uint64_t *offsets = nullptr;
 519                 uint32_t offsets_count;
 520                 struct lttng_kernel_abi_event_callsite callsite;
 521
 522                 /*
 523                  * This call allocates the offsets buffer. This buffer must be freed
 524                  * by the caller
 525                  */
 526                 ret = extract_userspace_probe_offset_tracepoint_sdt(
 527                         location, uid, gid, &offsets, &offsets_count);
 528                 if (ret) {
 529                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 530                         goto end;
 531                 }
 532                 for (i = 0; i < offsets_count; i++) {
 533                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 534                         ret = kernctl_add_callsite(fd, &callsite);
 535                         if (ret) {
 536                                 WARN("Failed to add callsite to SDT userspace probe");
 537                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 538                                 free(offsets);
 539                                 goto end;
 540                         }
 541                 }
 542                 free(offsets);
 543                 break;
 544         }
 545         default:
 546                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 547                 goto end;
 548         }
 549 end:
 550         return ret;
 551 }
 552
 553 /*
 554  * Extract the offsets of the instrumentation point for the different lookup
 555  * methods.
 556  */
 557 static int userspace_probe_event_add_callsites(struct lttng_event *ev,
 558                                                struct ltt_kernel_session *session,
 559                                                int fd)
 560 {
 561         int ret;
 562         const struct lttng_userspace_probe_location *location = nullptr;
 563
 564         LTTNG_ASSERT(ev);
 565         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 566
 567         location = lttng_event_get_userspace_probe_location(ev);
 568         if (!location) {
 569                 ret = -1;
 570                 goto end;
 571         }
 572
 573         ret = userspace_probe_add_callsite(location, session->uid, session->gid, fd);
 574         if (ret) {
 575                 WARN("Failed to add callsite to userspace probe event '%s'", ev->name);
 576         }
 577
 578 end:
 579         return ret;
 580 }
 581
 582 /*
 583  * Extract the offsets of the instrumentation point for the different look-up
 584  * methods.
 585  */
 586 static int userspace_probe_event_rule_add_callsites(const struct lttng_event_rule *rule,
 587                                                     const struct lttng_credentials *creds,
 588                                                     int fd)
 589 {
 590         int ret;
 591         enum lttng_event_rule_status status;
 592         enum lttng_event_rule_type event_rule_type;
 593         const struct lttng_userspace_probe_location *location = nullptr;
 594
 595         LTTNG_ASSERT(rule);
 596         LTTNG_ASSERT(creds);
 597
 598         event_rule_type = lttng_event_rule_get_type(rule);
 599         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 600
 601         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 602         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 603                 ret = -1;
 604                 goto end;
 605         }
 606
 607         ret = userspace_probe_add_callsite(
 608                 location, lttng_credentials_get_uid(creds), lttng_credentials_get_gid(creds), fd);
 609         if (ret) {
 610                 WARN("Failed to add callsite to user space probe object: fd = %d", fd);
 611         }
 612
 613 end:
 614         return ret;
 615 }
 616
 617 /*
 618  * Create a kernel event, enable it to the kernel tracer and add it to the
 619  * channel event list of the kernel session.
 620  * We own filter_expression and filter.
 621  */
 622 int kernel_create_event(struct lttng_event *ev,
 623                         struct ltt_kernel_channel *channel,
 624                         char *filter_expression,
 625                         struct lttng_bytecode *filter)
 626 {
 627         int err, fd;
 628         enum lttng_error_code ret;
 629         struct ltt_kernel_event *event;
 630
 631         LTTNG_ASSERT(ev);
 632         LTTNG_ASSERT(channel);
 633
 634         /* We pass ownership of filter_expression and filter */
 635         ret = trace_kernel_create_event(ev, filter_expression, filter, &event);
 636         if (ret != LTTNG_OK) {
 637                 goto error;
 638         }
 639
 640         fd = kernctl_create_event(channel->fd, event->event);
 641         if (fd < 0) {
 642                 switch (-fd) {
 643                 case EEXIST:
 644                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 645                         break;
 646                 case ENOSYS:
 647                         WARN("Event type not implemented");
 648                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 649                         break;
 650                 case ENOENT:
 651                         WARN("Event %s not found!", ev->name);
 652                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 653                         break;
 654                 default:
 655                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 656                         PERROR("create event ioctl");
 657                 }
 658                 goto free_event;
 659         }
 660
 661         event->type = ev->type;
 662         event->fd = fd;
 663         /* Prevent fd duplication after execlp() */
 664         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 665         if (err < 0) {
 666                 PERROR("fcntl session fd");
 667         }
 668
 669         if (filter) {
 670                 err = kernctl_filter(event->fd, filter);
 671                 if (err < 0) {
 672                         switch (-err) {
 673                         case ENOMEM:
 674                                 ret = LTTNG_ERR_FILTER_NOMEM;
 675                                 break;
 676                         default:
 677                                 ret = LTTNG_ERR_FILTER_INVAL;
 678                                 break;
 679                         }
 680                         goto filter_error;
 681                 }
 682         }
 683
 684         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 685                 ret = (lttng_error_code) userspace_probe_event_add_callsites(
 686                         ev, channel->session, event->fd);
 687                 if (ret) {
 688                         goto add_callsite_error;
 689                 }
 690         }
 691
 692         err = kernctl_enable(event->fd);
 693         if (err < 0) {
 694                 switch (-err) {
 695                 case EEXIST:
 696                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 697                         break;
 698                 default:
 699                         PERROR("enable kernel event");
 700                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 701                         break;
 702                 }
 703                 goto enable_error;
 704         }
 705
 706         /* Add event to event list */
 707         cds_list_add(&event->list, &channel->events_list.head);
 708         channel->event_count++;
 709
 710         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 711
 712         return 0;
 713
 714 add_callsite_error:
 715 enable_error:
 716 filter_error:
 717 {
 718         int closeret;
 719
 720         closeret = close(event->fd);
 721         if (closeret) {
 722                 PERROR("close event fd");
 723         }
 724 }
 725 free_event:
 726         free(event);
 727 error:
 728         return ret;
 729 }
 730
 731 /*
 732  * Disable a kernel channel.
 733  */
 734 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 735 {
 736         int ret;
 737
 738         LTTNG_ASSERT(chan);
 739
 740         ret = kernctl_disable(chan->fd);
 741         if (ret < 0) {
 742                 PERROR("disable chan ioctl");
 743                 goto error;
 744         }
 745
 746         chan->enabled = false;
 747         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 748             chan->channel->name,
 749             chan->fd,
 750             chan->key);
 751
 752         return 0;
 753
 754 error:
 755         return ret;
 756 }
 757
 758 /*
 759  * Enable a kernel channel.
 760  */
 761 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 762 {
 763         int ret;
 764
 765         LTTNG_ASSERT(chan);
 766
 767         ret = kernctl_enable(chan->fd);
 768         if (ret < 0 && ret != -EEXIST) {
 769                 PERROR("Enable kernel chan");
 770                 goto error;
 771         }
 772
 773         chan->enabled = true;
 774         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 775             chan->channel->name,
 776             chan->fd,
 777             chan->key);
 778
 779         return 0;
 780
 781 error:
 782         return ret;
 783 }
 784
 785 /*
 786  * Enable a kernel event.
 787  */
 788 int kernel_enable_event(struct ltt_kernel_event *event)
 789 {
 790         int ret;
 791
 792         LTTNG_ASSERT(event);
 793
 794         ret = kernctl_enable(event->fd);
 795         if (ret < 0) {
 796                 switch (-ret) {
 797                 case EEXIST:
 798                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 799                         break;
 800                 default:
 801                         PERROR("enable kernel event");
 802                         break;
 803                 }
 804                 goto error;
 805         }
 806
 807         event->enabled = true;
 808         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 809
 810         return 0;
 811
 812 error:
 813         return ret;
 814 }
 815
 816 /*
 817  * Disable a kernel event.
 818  */
 819 int kernel_disable_event(struct ltt_kernel_event *event)
 820 {
 821         int ret;
 822
 823         LTTNG_ASSERT(event);
 824
 825         ret = kernctl_disable(event->fd);
 826         if (ret < 0) {
 827                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 828                        event->event->name,
 829                        event->fd);
 830                 goto error;
 831         }
 832
 833         event->enabled = false;
 834         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 835
 836         return 0;
 837
 838 error:
 839         return ret;
 840 }
 841
 842 /*
 843  * Disable a kernel event notifier.
 844  */
 845 static int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 846 {
 847         int ret;
 848
 849         LTTNG_ASSERT(event);
 850
 851         const lttng::urcu::read_lock_guard read_lock;
 852         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 853
 854         ret = kernctl_disable(event->fd);
 855         if (ret < 0) {
 856                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 857                        event->fd,
 858                        event->token);
 859                 goto error;
 860         }
 861
 862         event->enabled = false;
 863         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64, event->fd, event->token);
 864
 865 error:
 866         return ret;
 867 }
 868
 869 static struct process_attr_tracker *
 870 _kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 871                                  enum lttng_process_attr process_attr)
 872 {
 873         switch (process_attr) {
 874         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 875                 return session->tracker_pid;
 876         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 877                 return session->tracker_vpid;
 878         case LTTNG_PROCESS_ATTR_USER_ID:
 879                 return session->tracker_uid;
 880         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 881                 return session->tracker_vuid;
 882         case LTTNG_PROCESS_ATTR_GROUP_ID:
 883                 return session->tracker_gid;
 884         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 885                 return session->tracker_vgid;
 886         default:
 887                 return nullptr;
 888         }
 889 }
 890
 891 const struct process_attr_tracker *
 892 kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 893                                 enum lttng_process_attr process_attr)
 894 {
 895         return (const struct process_attr_tracker *) _kernel_get_process_attr_tracker(session,
 896                                                                                       process_attr);
 897 }
 898
 899 enum lttng_error_code
 900 kernel_process_attr_tracker_set_tracking_policy(struct ltt_kernel_session *session,
 901                                                 enum lttng_process_attr process_attr,
 902                                                 enum lttng_tracking_policy policy)
 903 {
 904         int ret;
 905         enum lttng_error_code ret_code = LTTNG_OK;
 906         struct process_attr_tracker *tracker =
 907                 _kernel_get_process_attr_tracker(session, process_attr);
 908         enum lttng_tracking_policy previous_policy;
 909
 910         if (!tracker) {
 911                 ret_code = LTTNG_ERR_INVALID;
 912                 goto end;
 913         }
 914
 915         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 916         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 917         if (ret) {
 918                 ret_code = LTTNG_ERR_UNK;
 919                 goto end;
 920         }
 921
 922         if (previous_policy == policy) {
 923                 goto end;
 924         }
 925
 926         switch (policy) {
 927         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 928                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 929                         /*
 930                          * Maintain a special case for the process ID process
 931                          * attribute tracker as it was the only supported
 932                          * attribute prior to 2.12.
 933                          */
 934                         ret = kernctl_track_pid(session->fd, -1);
 935                 } else {
 936                         ret = kernctl_track_id(session->fd, process_attr, -1);
 937                 }
 938                 break;
 939         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 940         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 941                 /* fall-through. */
 942                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 943                         /*
 944                          * Maintain a special case for the process ID process
 945                          * attribute tracker as it was the only supported
 946                          * attribute prior to 2.12.
 947                          */
 948                         ret = kernctl_untrack_pid(session->fd, -1);
 949                 } else {
 950                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 951                 }
 952                 break;
 953         default:
 954                 abort();
 955         }
 956         /* kern-ctl error handling */
 957         switch (-ret) {
 958         case 0:
 959                 ret_code = LTTNG_OK;
 960                 break;
 961         case EINVAL:
 962                 ret_code = LTTNG_ERR_INVALID;
 963                 break;
 964         case ENOMEM:
 965                 ret_code = LTTNG_ERR_NOMEM;
 966                 break;
 967         case EEXIST:
 968                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 969                 break;
 970         default:
 971                 ret_code = LTTNG_ERR_UNK;
 972                 break;
 973         }
 974 end:
 975         return ret_code;
 976 }
 977
 978 enum lttng_error_code
 979 kernel_process_attr_tracker_inclusion_set_add_value(struct ltt_kernel_session *session,
 980                                                     enum lttng_process_attr process_attr,
 981                                                     const struct process_attr_value *value)
 982 {
 983         int ret, integral_value;
 984         enum lttng_error_code ret_code;
 985         struct process_attr_tracker *tracker;
 986         enum process_attr_tracker_status status;
 987
 988         /*
 989          * Convert process attribute tracker value to the integral
 990          * representation required by the kern-ctl API.
 991          */
 992         switch (process_attr) {
 993         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 994         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 995                 integral_value = (int) value->value.pid;
 996                 break;
 997         case LTTNG_PROCESS_ATTR_USER_ID:
 998         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 999                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1000                         uid_t uid;
1001
1002                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1003                         if (ret_code != LTTNG_OK) {
1004                                 goto end;
1005                         }
1006                         integral_value = (int) uid;
1007                 } else {
1008                         integral_value = (int) value->value.uid;
1009                 }
1010                 break;
1011         case LTTNG_PROCESS_ATTR_GROUP_ID:
1012         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1013                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1014                         gid_t gid;
1015
1016                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1017                         if (ret_code != LTTNG_OK) {
1018                                 goto end;
1019                         }
1020                         integral_value = (int) gid;
1021                 } else {
1022                         integral_value = (int) value->value.gid;
1023                 }
1024                 break;
1025         default:
1026                 ret_code = LTTNG_ERR_INVALID;
1027                 goto end;
1028         }
1029
1030         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1031         if (!tracker) {
1032                 ret_code = LTTNG_ERR_INVALID;
1033                 goto end;
1034         }
1035
1036         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1037         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1038                 switch (status) {
1039                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1040                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1041                         break;
1042                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1043                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1044                         break;
1045                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1046                 default:
1047                         ret_code = LTTNG_ERR_UNK;
1048                         break;
1049                 }
1050                 goto end;
1051         }
1052
1053         DBG("Kernel track %s %d for session id %" PRIu64,
1054             lttng_process_attr_to_string(process_attr),
1055             integral_value,
1056             session->id);
1057         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1058                 /*
1059                  * Maintain a special case for the process ID process attribute
1060                  * tracker as it was the only supported attribute prior to 2.12.
1061                  */
1062                 ret = kernctl_track_pid(session->fd, integral_value);
1063         } else {
1064                 ret = kernctl_track_id(session->fd, process_attr, integral_value);
1065         }
1066         if (ret == 0) {
1067                 ret_code = LTTNG_OK;
1068                 goto end;
1069         }
1070
1071         kernel_wait_quiescent();
1072
1073         /* kern-ctl error handling */
1074         switch (-ret) {
1075         case 0:
1076                 ret_code = LTTNG_OK;
1077                 break;
1078         case EINVAL:
1079                 ret_code = LTTNG_ERR_INVALID;
1080                 break;
1081         case ENOMEM:
1082                 ret_code = LTTNG_ERR_NOMEM;
1083                 break;
1084         case EEXIST:
1085                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1086                 break;
1087         default:
1088                 ret_code = LTTNG_ERR_UNK;
1089                 break;
1090         }
1091
1092         /* Attempt to remove the value from the tracker. */
1093         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1094         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1095                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1096                     lttng_process_attr_to_string(process_attr),
1097                     integral_value);
1098         }
1099 end:
1100         return ret_code;
1101 }
1102
1103 enum lttng_error_code
1104 kernel_process_attr_tracker_inclusion_set_remove_value(struct ltt_kernel_session *session,
1105                                                        enum lttng_process_attr process_attr,
1106                                                        const struct process_attr_value *value)
1107 {
1108         int ret, integral_value;
1109         enum lttng_error_code ret_code;
1110         struct process_attr_tracker *tracker;
1111         enum process_attr_tracker_status status;
1112
1113         /*
1114          * Convert process attribute tracker value to the integral
1115          * representation required by the kern-ctl API.
1116          */
1117         switch (process_attr) {
1118         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1119         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1120                 integral_value = (int) value->value.pid;
1121                 break;
1122         case LTTNG_PROCESS_ATTR_USER_ID:
1123         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1124                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1125                         uid_t uid;
1126
1127                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1128                         if (ret_code != LTTNG_OK) {
1129                                 goto end;
1130                         }
1131                         integral_value = (int) uid;
1132                 } else {
1133                         integral_value = (int) value->value.uid;
1134                 }
1135                 break;
1136         case LTTNG_PROCESS_ATTR_GROUP_ID:
1137         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1138                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1139                         gid_t gid;
1140
1141                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1142                         if (ret_code != LTTNG_OK) {
1143                                 goto end;
1144                         }
1145                         integral_value = (int) gid;
1146                 } else {
1147                         integral_value = (int) value->value.gid;
1148                 }
1149                 break;
1150         default:
1151                 ret_code = LTTNG_ERR_INVALID;
1152                 goto end;
1153         }
1154
1155         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1156         if (!tracker) {
1157                 ret_code = LTTNG_ERR_INVALID;
1158                 goto end;
1159         }
1160
1161         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1162         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1163                 switch (status) {
1164                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1165                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1166                         break;
1167                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1168                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1169                         break;
1170                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1171                 default:
1172                         ret_code = LTTNG_ERR_UNK;
1173                         break;
1174                 }
1175                 goto end;
1176         }
1177
1178         DBG("Kernel track %s %d for session id %" PRIu64,
1179             lttng_process_attr_to_string(process_attr),
1180             integral_value,
1181             session->id);
1182         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1183                 /*
1184                  * Maintain a special case for the process ID process attribute
1185                  * tracker as it was the only supported attribute prior to 2.12.
1186                  */
1187                 ret = kernctl_untrack_pid(session->fd, integral_value);
1188         } else {
1189                 ret = kernctl_untrack_id(session->fd, process_attr, integral_value);
1190         }
1191         if (ret == 0) {
1192                 ret_code = LTTNG_OK;
1193                 goto end;
1194         }
1195         kernel_wait_quiescent();
1196
1197         /* kern-ctl error handling */
1198         switch (-ret) {
1199         case 0:
1200                 ret_code = LTTNG_OK;
1201                 break;
1202         case EINVAL:
1203                 ret_code = LTTNG_ERR_INVALID;
1204                 break;
1205         case ENOMEM:
1206                 ret_code = LTTNG_ERR_NOMEM;
1207                 break;
1208         case ENOENT:
1209                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1210                 break;
1211         default:
1212                 ret_code = LTTNG_ERR_UNK;
1213                 break;
1214         }
1215
1216         /* Attempt to add the value to the tracker. */
1217         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1218         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1219                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1220                     lttng_process_attr_to_string(process_attr),
1221                     integral_value);
1222         }
1223 end:
1224         return ret_code;
1225 }
1226
1227 /*
1228  * Create kernel metadata, open from the kernel tracer and add it to the
1229  * kernel session.
1230  */
1231 int kernel_open_metadata(struct ltt_kernel_session *session)
1232 {
1233         int ret;
1234         struct ltt_kernel_metadata *lkm = nullptr;
1235
1236         LTTNG_ASSERT(session);
1237
1238         /* Allocate kernel metadata */
1239         lkm = trace_kernel_create_metadata();
1240         if (lkm == nullptr) {
1241                 goto error;
1242         }
1243
1244         /* Kernel tracer metadata creation */
1245         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1246         if (ret < 0) {
1247                 goto error_open;
1248         }
1249
1250         lkm->fd = ret;
1251         lkm->key = ++next_kernel_channel_key;
1252         /* Prevent fd duplication after execlp() */
1253         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1254         if (ret < 0) {
1255                 PERROR("fcntl session fd");
1256         }
1257
1258         session->metadata = lkm;
1259
1260         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1261
1262         return 0;
1263
1264 error_open:
1265         trace_kernel_destroy_metadata(lkm);
1266 error:
1267         return -1;
1268 }
1269
1270 /*
1271  * Start tracing session.
1272  */
1273 int kernel_start_session(struct ltt_kernel_session *session)
1274 {
1275         int ret;
1276
1277         LTTNG_ASSERT(session);
1278
1279         ret = kernctl_start_session(session->fd);
1280         if (ret < 0) {
1281                 PERROR("ioctl start session");
1282                 goto error;
1283         }
1284
1285         DBG("Kernel session started");
1286
1287         return 0;
1288
1289 error:
1290         return ret;
1291 }
1292
1293 /*
1294  * Make a kernel wait to make sure in-flight probe have completed.
1295  */
1296 void kernel_wait_quiescent()
1297 {
1298         int ret;
1299         const int fd = kernel_tracer_fd;
1300
1301         DBG("Kernel quiescent wait on %d", fd);
1302
1303         ret = kernctl_wait_quiescent(fd);
1304         if (ret < 0) {
1305                 PERROR("wait quiescent ioctl");
1306                 ERR("Kernel quiescent wait failed");
1307         }
1308 }
1309
1310 /*
1311  *  Force flush buffer of metadata.
1312  */
1313 int kernel_metadata_flush_buffer(int fd)
1314 {
1315         int ret;
1316
1317         DBG("Kernel flushing metadata buffer on fd %d", fd);
1318
1319         ret = kernctl_buffer_flush(fd);
1320         if (ret < 0) {
1321                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1322         }
1323
1324         return 0;
1325 }
1326
1327 /*
1328  * Force flush buffer for channel.
1329  */
1330 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1331 {
1332         int ret;
1333         struct ltt_kernel_stream *stream;
1334
1335         LTTNG_ASSERT(channel);
1336
1337         DBG("Flush buffer for channel %s", channel->channel->name);
1338
1339         cds_list_for_each_entry (stream, &channel->stream_list.head, list) {
1340                 DBG("Flushing channel stream %d", stream->fd);
1341                 ret = kernctl_buffer_flush(stream->fd);
1342                 if (ret < 0) {
1343                         PERROR("ioctl");
1344                         ERR("Fail to flush buffer for stream %d (ret: %d)", stream->fd, ret);
1345                 }
1346         }
1347
1348         return 0;
1349 }
1350
1351 /*
1352  * Stop tracing session.
1353  */
1354 int kernel_stop_session(struct ltt_kernel_session *session)
1355 {
1356         int ret;
1357
1358         LTTNG_ASSERT(session);
1359
1360         ret = kernctl_stop_session(session->fd);
1361         if (ret < 0) {
1362                 goto error;
1363         }
1364
1365         DBG("Kernel session stopped");
1366
1367         return 0;
1368
1369 error:
1370         return ret;
1371 }
1372
1373 /*
1374  * Open stream of channel, register it to the kernel tracer and add it
1375  * to the stream list of the channel.
1376  *
1377  * Note: given that the streams may appear in random order wrt CPU
1378  * number (e.g. cpu hotplug), the index value of the stream number in
1379  * the stream name is not necessarily linked to the CPU number.
1380  *
1381  * Return the number of created stream. Else, a negative value.
1382  */
1383 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1384 {
1385         int ret;
1386         struct ltt_kernel_stream *lks;
1387
1388         LTTNG_ASSERT(channel);
1389
1390         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1391                 lks = trace_kernel_create_stream(channel->channel->name, channel->stream_count);
1392                 if (lks == nullptr) {
1393                         ret = close(ret);
1394                         if (ret) {
1395                                 PERROR("close");
1396                         }
1397                         goto error;
1398                 }
1399
1400                 lks->fd = ret;
1401                 /* Prevent fd duplication after execlp() */
1402                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1403                 if (ret < 0) {
1404                         PERROR("fcntl session fd");
1405                 }
1406
1407                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1408                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1409
1410                 /* Add stream to channel stream list */
1411                 cds_list_add(&lks->list, &channel->stream_list.head);
1412                 channel->stream_count++;
1413
1414                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd, lks->state);
1415         }
1416
1417         return channel->stream_count;
1418
1419 error:
1420         return -1;
1421 }
1422
1423 /*
1424  * Open the metadata stream and set it to the kernel session.
1425  */
1426 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1427 {
1428         int ret;
1429
1430         LTTNG_ASSERT(session);
1431
1432         ret = kernctl_create_stream(session->metadata->fd);
1433         if (ret < 0) {
1434                 PERROR("kernel create metadata stream");
1435                 goto error;
1436         }
1437
1438         DBG("Kernel metadata stream created (fd: %d)", ret);
1439         session->metadata_stream_fd = ret;
1440         /* Prevent fd duplication after execlp() */
1441         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1442         if (ret < 0) {
1443                 PERROR("fcntl session fd");
1444         }
1445
1446         return 0;
1447
1448 error:
1449         return -1;
1450 }
1451
1452 /*
1453  * Get the event list from the kernel tracer and return the number of elements.
1454  */
1455 ssize_t kernel_list_events(struct lttng_event **events)
1456 {
1457         int fd, ret;
1458         char *event;
1459         size_t nbmem, count = 0;
1460         FILE *fp;
1461         struct lttng_event *elist;
1462
1463         LTTNG_ASSERT(events);
1464
1465         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1466         if (fd < 0) {
1467                 PERROR("kernel tracepoint list");
1468                 goto error;
1469         }
1470
1471         fp = fdopen(fd, "r");
1472         if (fp == nullptr) {
1473                 PERROR("kernel tracepoint list fdopen");
1474                 goto error_fp;
1475         }
1476
1477         /*
1478          * Init memory size counter
1479          * See kernel-ctl.h for explanation of this value
1480          */
1481         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1482         elist = calloc<lttng_event>(nbmem);
1483         if (elist == nullptr) {
1484                 PERROR("alloc list events");
1485                 count = -ENOMEM;
1486                 goto end;
1487         }
1488
1489         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1490                 if (count >= nbmem) {
1491                         struct lttng_event *new_elist;
1492                         size_t new_nbmem;
1493
1494                         new_nbmem = nbmem << 1;
1495                         DBG("Reallocating event list from %zu to %zu bytes", nbmem, new_nbmem);
1496                         new_elist = (lttng_event *) realloc(elist,
1497                                                             new_nbmem * sizeof(struct lttng_event));
1498                         if (new_elist == nullptr) {
1499                                 PERROR("realloc list events");
1500                                 free(event);
1501                                 free(elist);
1502                                 count = -ENOMEM;
1503                                 goto end;
1504                         }
1505                         /* Zero the new memory */
1506                         memset(new_elist + nbmem,
1507                                0,
1508                                (new_nbmem - nbmem) * sizeof(struct lttng_event));
1509                         nbmem = new_nbmem;
1510                         elist = new_elist;
1511                 }
1512                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1513                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1514                 elist[count].enabled = -1;
1515                 count++;
1516                 free(event);
1517         }
1518
1519         *events = elist;
1520         DBG("Kernel list events done (%zu events)", count);
1521 end:
1522         ret = fclose(fp); /* closes both fp and fd */
1523         if (ret) {
1524                 PERROR("fclose");
1525         }
1526         return count;
1527
1528 error_fp:
1529         ret = close(fd);
1530         if (ret) {
1531                 PERROR("close");
1532         }
1533 error:
1534         return -1;
1535 }
1536
1537 /*
1538  * Get kernel version and validate it.
1539  */
1540 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1541                             struct lttng_kernel_abi_tracer_abi_version *abi_version)
1542 {
1543         int ret;
1544
1545         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1546         if (ret < 0) {
1547                 ERR("Failed to retrieve the lttng-modules version");
1548                 goto error;
1549         }
1550
1551         /* Validate version */
1552         if (version->major != VERSION_MAJOR) {
1553                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1554                     version->major,
1555                     VERSION_MAJOR);
1556                 goto error_version;
1557         }
1558         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1559         if (ret < 0) {
1560                 ERR("Failed to retrieve lttng-modules ABI version");
1561                 goto error;
1562         }
1563         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1564                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1565                     abi_version->major,
1566                     abi_version->minor,
1567                     LTTNG_KERNEL_ABI_MAJOR_VERSION);
1568                 goto error;
1569         }
1570         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1571              version->major,
1572              version->minor,
1573              abi_version->major,
1574              abi_version->minor);
1575         return 0;
1576
1577 error_version:
1578         ret = -1;
1579
1580 error:
1581         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1582         return ret;
1583 }
1584
1585 /*
1586  * Kernel work-arounds called at the start of sessiond main().
1587  */
1588 int init_kernel_workarounds()
1589 {
1590         int ret;
1591         FILE *fp;
1592
1593         /*
1594          * boot_id needs to be read once before being used concurrently
1595          * to deal with a Linux kernel race. A fix is proposed for
1596          * upstream, but the work-around is needed for older kernels.
1597          */
1598         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1599         if (!fp) {
1600                 goto end_boot_id;
1601         }
1602         while (!feof(fp)) {
1603                 char buf[37] = "";
1604
1605                 ret = fread(buf, 1, sizeof(buf), fp);
1606                 if (ret < 0) {
1607                         /* Ignore error, we don't really care */
1608                 }
1609         }
1610         ret = fclose(fp);
1611         if (ret) {
1612                 PERROR("fclose");
1613         }
1614 end_boot_id:
1615         return 0;
1616 }
1617
1618 /*
1619  * Teardown of a kernel session, keeping data required by destroy notifiers.
1620  */
1621 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1622 {
1623         struct lttng_trace_chunk *trace_chunk;
1624
1625         if (ksess == nullptr) {
1626                 DBG3("No kernel session when tearing down session");
1627                 return;
1628         }
1629
1630         DBG("Tearing down kernel session");
1631         trace_chunk = ksess->current_trace_chunk;
1632
1633         /*
1634          * Destroy channels on the consumer if at least one FD has been sent and we
1635          * are in no output mode because the streams are in *no* monitor mode so we
1636          * have to send a command to clean them up or else they leaked.
1637          */
1638         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1639                 int ret;
1640                 struct consumer_socket *socket;
1641                 struct lttng_ht_iter iter;
1642
1643                 /* For each consumer socket. */
1644                 const lttng::urcu::read_lock_guard read_lock;
1645
1646                 cds_lfht_for_each_entry (
1647                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1648                         struct ltt_kernel_channel *chan;
1649
1650                         /* For each channel, ask the consumer to destroy it. */
1651                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1652                                 ret = kernel_consumer_destroy_channel(socket, chan);
1653                                 if (ret < 0) {
1654                                         /* Consumer is probably dead. Use next socket. */
1655                                         continue;
1656                                 }
1657                         }
1658                 }
1659         }
1660
1661         /* Close any relayd session */
1662         consumer_output_send_destroy_relayd(ksess->consumer);
1663
1664         trace_kernel_destroy_session(ksess);
1665         lttng_trace_chunk_put(trace_chunk);
1666 }
1667
1668 /* Teardown of data required by destroy notifiers. */
1669 void kernel_free_session(struct ltt_kernel_session *ksess)
1670 {
1671         if (ksess == nullptr) {
1672                 return;
1673         }
1674         trace_kernel_free_session(ksess);
1675 }
1676
1677 /*
1678  * Destroy a kernel channel object. It does not do anything on the tracer side.
1679  */
1680 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1681 {
1682         struct ltt_kernel_session *ksess = nullptr;
1683
1684         LTTNG_ASSERT(kchan);
1685         LTTNG_ASSERT(kchan->channel);
1686
1687         DBG3("Kernel destroy channel %s", kchan->channel->name);
1688
1689         /* Update channel count of associated session. */
1690         if (kchan->session) {
1691                 /* Keep pointer reference so we can update it after the destroy. */
1692                 ksess = kchan->session;
1693         }
1694
1695         trace_kernel_destroy_channel(kchan);
1696
1697         /*
1698          * At this point the kernel channel is not visible anymore. This is safe
1699          * since in order to work on a visible kernel session, the tracing session
1700          * lock (ltt_session.lock) MUST be acquired.
1701          */
1702         if (ksess) {
1703                 ksess->channel_count--;
1704         }
1705 }
1706
1707 /*
1708  * Take a snapshot for a given kernel session.
1709  *
1710  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1711  */
1712 enum lttng_error_code kernel_snapshot_record(struct ltt_kernel_session *ksess,
1713                                              const struct consumer_output *output,
1714                                              uint64_t nb_packets_per_stream)
1715 {
1716         int err, ret, saved_metadata_fd;
1717         enum lttng_error_code status = LTTNG_OK;
1718         struct consumer_socket *socket;
1719         struct lttng_ht_iter iter;
1720         struct ltt_kernel_metadata *saved_metadata;
1721         char *trace_path = nullptr;
1722         size_t consumer_path_offset = 0;
1723
1724         LTTNG_ASSERT(ksess);
1725         LTTNG_ASSERT(ksess->consumer);
1726         LTTNG_ASSERT(output);
1727
1728         DBG("Kernel snapshot record started");
1729
1730         /* Save current metadata since the following calls will change it. */
1731         saved_metadata = ksess->metadata;
1732         saved_metadata_fd = ksess->metadata_stream_fd;
1733
1734         ret = kernel_open_metadata(ksess);
1735         if (ret < 0) {
1736                 status = LTTNG_ERR_KERN_META_FAIL;
1737                 goto error;
1738         }
1739
1740         ret = kernel_open_metadata_stream(ksess);
1741         if (ret < 0) {
1742                 status = LTTNG_ERR_KERN_META_FAIL;
1743                 goto error_open_stream;
1744         }
1745
1746         trace_path = setup_channel_trace_path(ksess->consumer, "", &consumer_path_offset);
1747         if (!trace_path) {
1748                 status = LTTNG_ERR_INVALID;
1749                 goto error;
1750         }
1751
1752         {
1753                 /* Send metadata to consumer and snapshot everything. */
1754                 const lttng::urcu::read_lock_guard read_lock;
1755
1756                 cds_lfht_for_each_entry (output->socks->ht, &iter.iter, socket, node.node) {
1757                         struct ltt_kernel_channel *chan;
1758
1759                         pthread_mutex_lock(socket->lock);
1760                         /* This stream must not be monitored by the consumer. */
1761                         ret = kernel_consumer_add_metadata(socket, ksess, 0);
1762                         pthread_mutex_unlock(socket->lock);
1763                         if (ret < 0) {
1764                                 status = LTTNG_ERR_KERN_META_FAIL;
1765                                 goto error_consumer;
1766                         }
1767
1768                         /* For each channel, ask the consumer to snapshot it. */
1769                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1770                                 status =
1771                                         consumer_snapshot_channel(socket,
1772                                                                   chan->key,
1773                                                                   output,
1774                                                                   0,
1775                                                                   &trace_path[consumer_path_offset],
1776                                                                   nb_packets_per_stream);
1777                                 if (status != LTTNG_OK) {
1778                                         (void) kernel_consumer_destroy_metadata(socket,
1779                                                                                 ksess->metadata);
1780                                         goto error_consumer;
1781                                 }
1782                         }
1783
1784                         /* Snapshot metadata, */
1785                         status = consumer_snapshot_channel(socket,
1786                                                            ksess->metadata->key,
1787                                                            output,
1788                                                            1,
1789                                                            &trace_path[consumer_path_offset],
1790                                                            0);
1791                         if (status != LTTNG_OK) {
1792                                 goto error_consumer;
1793                         }
1794
1795                         /*
1796                          * The metadata snapshot is done, ask the consumer to destroy it since
1797                          * it's not monitored on the consumer side.
1798                          */
1799                         (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1800                 }
1801         }
1802
1803 error_consumer:
1804         /* Close newly opened metadata stream. It's now on the consumer side. */
1805         err = close(ksess->metadata_stream_fd);
1806         if (err < 0) {
1807                 PERROR("close snapshot kernel");
1808         }
1809
1810 error_open_stream:
1811         trace_kernel_destroy_metadata(ksess->metadata);
1812 error:
1813         /* Restore metadata state.*/
1814         ksess->metadata = saved_metadata;
1815         ksess->metadata_stream_fd = saved_metadata_fd;
1816         free(trace_path);
1817         return status;
1818 }
1819
1820 /*
1821  * Get the syscall mask array from the kernel tracer.
1822  *
1823  * Return 0 on success else a negative value. In both case, syscall_mask should
1824  * be freed.
1825  */
1826 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1827 {
1828         LTTNG_ASSERT(syscall_mask);
1829         LTTNG_ASSERT(nr_bits);
1830
1831         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1832 }
1833
1834 static int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1835 {
1836         int ret;
1837         struct lttng_kernel_abi_tracer_abi_version abi;
1838
1839         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1840         if (ret < 0) {
1841                 ERR("Failed to retrieve lttng-modules ABI version");
1842                 goto error;
1843         }
1844
1845         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1846 error:
1847         return ret;
1848 }
1849
1850 /*
1851  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1852  * version number.
1853  *
1854  * Return 1 on success, 0 when feature is not supported, negative value in case
1855  * of errors.
1856  */
1857 int kernel_supports_ring_buffer_snapshot_sample_positions()
1858 {
1859         /*
1860          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1861          */
1862         return kernel_tracer_abi_greater_or_equal(2, 3);
1863 }
1864
1865 /*
1866  * Check for the support of the packet sequence number via abi version number.
1867  *
1868  * Return 1 on success, 0 when feature is not supported, negative value in case
1869  * of errors.
1870  */
1871 int kernel_supports_ring_buffer_packet_sequence_number()
1872 {
1873         /*
1874          * Packet sequence number was introduced in LTTng 2.8,
1875          * lttng-modules ABI 2.1.
1876          */
1877         return kernel_tracer_abi_greater_or_equal(2, 1);
1878 }
1879
1880 /*
1881  * Check for the support of event notifiers via abi version number.
1882  *
1883  * Return 1 on success, 0 when feature is not supported, negative value in case
1884  * of errors.
1885  */
1886 int kernel_supports_event_notifiers()
1887 {
1888         /*
1889          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1890          */
1891         return kernel_tracer_abi_greater_or_equal(2, 6);
1892 }
1893
1894 /*
1895  * Rotate a kernel session.
1896  *
1897  * Return LTTNG_OK on success or else an LTTng error code.
1898  */
1899 enum lttng_error_code kernel_rotate_session(const ltt_session::locked_ref& session)
1900 {
1901         int ret;
1902         enum lttng_error_code status = LTTNG_OK;
1903         struct consumer_socket *socket;
1904         struct lttng_ht_iter iter;
1905         struct ltt_kernel_session *ksess = session->kernel_session;
1906
1907         LTTNG_ASSERT(ksess);
1908         LTTNG_ASSERT(ksess->consumer);
1909
1910         DBG("Rotate kernel session %s started (session %" PRIu64 ")", session->name, session->id);
1911
1912         {
1913                 /*
1914                  * Note that this loop will end after one iteration given that there is
1915                  * only one kernel consumer.
1916                  */
1917                 const lttng::urcu::read_lock_guard read_lock;
1918
1919                 cds_lfht_for_each_entry (
1920                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1921                         struct ltt_kernel_channel *chan;
1922
1923                         /* For each channel, ask the consumer to rotate it. */
1924                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1925                                 DBG("Rotate kernel channel %" PRIu64 ", session %s",
1926                                     chan->key,
1927                                     session->name);
1928                                 ret = consumer_rotate_channel(socket,
1929                                                               chan->key,
1930                                                               ksess->consumer,
1931                                                               /* is_metadata_channel */ false);
1932                                 if (ret < 0) {
1933                                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1934                                         goto error;
1935                                 }
1936                         }
1937
1938                         /*
1939                          * Rotate the metadata channel.
1940                          */
1941                         ret = consumer_rotate_channel(socket,
1942                                                       ksess->metadata->key,
1943                                                       ksess->consumer,
1944                                                       /* is_metadata_channel */ true);
1945                         if (ret < 0) {
1946                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1947                                 goto error;
1948                         }
1949                 }
1950         }
1951
1952 error:
1953         return status;
1954 }
1955
1956 enum lttng_error_code kernel_create_channel_subdirectories(const struct ltt_kernel_session *ksess)
1957 {
1958         enum lttng_error_code ret = LTTNG_OK;
1959         enum lttng_trace_chunk_status chunk_status;
1960
1961         const lttng::urcu::read_lock_guard read_lock;
1962         LTTNG_ASSERT(ksess->current_trace_chunk);
1963
1964         /*
1965          * Create the index subdirectory which will take care
1966          * of implicitly creating the channel's path.
1967          */
1968         chunk_status = lttng_trace_chunk_create_subdirectory(
1969                 ksess->current_trace_chunk, DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1970         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1971                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1972                 goto error;
1973         }
1974 error:
1975         return ret;
1976 }
1977
1978 /*
1979  * Get current kernel tracer status
1980  */
1981 enum lttng_kernel_tracer_status get_kernel_tracer_status()
1982 {
1983         if (!kernel_tracer_status) {
1984                 return LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN;
1985         }
1986
1987         return *kernel_tracer_status;
1988 }
1989
1990 /*
1991  * Sets the kernel tracer status based on the positive errno code
1992  */
1993 void set_kernel_tracer_status_from_modules_ret(int code)
1994 {
1995         switch (code) {
1996         case ENOENT:
1997         {
1998                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
1999                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING);
2000                 break;
2001         }
2002         case ENOKEY:
2003         case EKEYEXPIRED:
2004         case EKEYREVOKED:
2005         case EKEYREJECTED:
2006         {
2007                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2008                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE);
2009                 break;
2010         }
2011         default:
2012         {
2013                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2014                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN);
2015                 break;
2016         }
2017         }
2018 }
2019
2020 /*
2021  * Setup necessary data for kernel tracer action.
2022  */
2023 int init_kernel_tracer()
2024 {
2025         int ret;
2026         const bool is_root = !getuid();
2027
2028         const auto log_status_on_exit = lttng::make_scope_exit([]() noexcept {
2029                 DBG_FMT("Kernel tracer status set to `{}`",
2030                         kernel_tracer_status_to_str(*kernel_tracer_status));
2031         });
2032
2033         /* Modprobe lttng kernel modules */
2034         ret = modprobe_lttng_control();
2035         if (ret < 0) {
2036                 set_kernel_tracer_status_from_modules_ret(-ret);
2037                 goto error;
2038         }
2039
2040         /* Open debugfs lttng */
2041         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
2042         if (kernel_tracer_fd < 0) {
2043                 DBG("Failed to open %s", module_proc_lttng);
2044                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2045                         LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG);
2046                 goto error_open;
2047         }
2048
2049         /* Validate kernel version */
2050         ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
2051         if (ret < 0) {
2052                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2053                         LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH);
2054                 goto error_version;
2055         }
2056
2057         ret = modprobe_lttng_data();
2058         if (ret < 0) {
2059                 set_kernel_tracer_status_from_modules_ret(-ret);
2060                 goto error_modules;
2061         }
2062
2063         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
2064         if (ret < 0) {
2065                 goto error_modules;
2066         }
2067         if (ret < 1) {
2068                 WARN("Kernel tracer does not support buffer monitoring. "
2069                      "The monitoring timer of channels in the kernel domain "
2070                      "will be set to 0 (disabled).");
2071         }
2072
2073         ret = kernel_supports_event_notifiers();
2074         if (ret < 0) {
2075                 ERR("Failed to check for kernel tracer event notifier support");
2076                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2077                         LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2078                 goto error_modules;
2079         }
2080         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2081         if (ret < 0) {
2082                 /* This is not fatal. */
2083                 WARN("Failed to create kernel event notifier group");
2084                 kernel_tracer_event_notifier_group_fd = -1;
2085         } else {
2086                 enum event_notifier_error_accounting_status error_accounting_status;
2087                 enum lttng_error_code error_code_ret =
2088                         kernel_create_event_notifier_group_notification_fd(
2089                                 &kernel_tracer_event_notifier_group_notification_fd);
2090
2091                 if (error_code_ret != LTTNG_OK) {
2092                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2093                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2094                         goto error_modules;
2095                 }
2096
2097                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2098                         kernel_tracer_event_notifier_group_fd);
2099                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2100                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2101                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2102                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2103                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2104                         goto error_modules;
2105                 }
2106
2107                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2108                         DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
2109                 if (!kernel_token_to_event_notifier_rule_ht) {
2110                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2111                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2112                         goto error_token_ht;
2113                 }
2114         }
2115
2116         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2117             kernel_tracer_fd,
2118             kernel_tracer_event_notifier_group_fd,
2119             kernel_tracer_event_notifier_group_notification_fd);
2120
2121         ret = syscall_init_table(kernel_tracer_fd);
2122         if (ret < 0) {
2123                 ERR("Unable to populate syscall table. Syscall tracing won't "
2124                     "work for this session daemon.");
2125         }
2126
2127         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2128                 LTTNG_KERNEL_TRACER_STATUS_INITIALIZED);
2129         return 0;
2130
2131 error_version:
2132         modprobe_remove_lttng_control();
2133         ret = close(kernel_tracer_fd);
2134         if (ret) {
2135                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2136         }
2137
2138         kernel_tracer_fd = -1;
2139         return LTTNG_ERR_KERN_VERSION;
2140
2141 error_token_ht:
2142         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2143         if (ret) {
2144                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2145                        kernel_tracer_event_notifier_group_notification_fd);
2146         }
2147
2148         kernel_tracer_event_notifier_group_notification_fd = -1;
2149
2150 error_modules:
2151         ret = close(kernel_tracer_event_notifier_group_fd);
2152         if (ret) {
2153                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2154                        kernel_tracer_event_notifier_group_fd);
2155         }
2156
2157         kernel_tracer_event_notifier_group_fd = -1;
2158
2159         ret = close(kernel_tracer_fd);
2160         if (ret) {
2161                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2162         }
2163
2164         kernel_tracer_fd = -1;
2165
2166 error_open:
2167         modprobe_remove_lttng_control();
2168
2169 error:
2170         WARN("No kernel tracer available");
2171         kernel_tracer_fd = -1;
2172         if (!is_root) {
2173                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2174                         LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT);
2175                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2176         } else {
2177                 return LTTNG_ERR_KERN_NA;
2178         }
2179 }
2180
2181 void cleanup_kernel_tracer()
2182 {
2183         DBG2("Closing kernel event notifier group notification file descriptor");
2184         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2185                 int ret = notification_thread_command_remove_tracer_event_source(
2186                         the_notification_thread_handle,
2187                         kernel_tracer_event_notifier_group_notification_fd);
2188                 if (ret != LTTNG_OK) {
2189                         ERR("Failed to remove kernel event notifier notification from notification thread");
2190                 }
2191
2192                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2193                 if (ret) {
2194                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2195                                kernel_tracer_event_notifier_group_notification_fd);
2196                 }
2197
2198                 kernel_tracer_event_notifier_group_notification_fd = -1;
2199         }
2200
2201         if (kernel_token_to_event_notifier_rule_ht) {
2202                 const int ret = cds_lfht_destroy(kernel_token_to_event_notifier_rule_ht, nullptr);
2203                 LTTNG_ASSERT(ret == 0);
2204         }
2205
2206         DBG2("Closing kernel event notifier group file descriptor");
2207         if (kernel_tracer_event_notifier_group_fd >= 0) {
2208                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2209
2210                 if (ret) {
2211                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2212                                kernel_tracer_event_notifier_group_fd);
2213                 }
2214
2215                 kernel_tracer_event_notifier_group_fd = -1;
2216         }
2217
2218         DBG2("Closing kernel fd");
2219         if (kernel_tracer_fd >= 0) {
2220                 const int ret = close(kernel_tracer_fd);
2221
2222                 if (ret) {
2223                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2224                                kernel_tracer_fd);
2225                 }
2226
2227                 kernel_tracer_fd = -1;
2228         }
2229
2230         kernel_tracer_status = nonstd::nullopt;
2231         free(syscall_table);
2232 }
2233
2234 bool kernel_tracer_is_initialized()
2235 {
2236         return kernel_tracer_fd >= 0;
2237 }
2238
2239 /*
2240  *  Clear a kernel session.
2241  *
2242  * Return LTTNG_OK on success or else an LTTng error code.
2243  */
2244 enum lttng_error_code kernel_clear_session(const ltt_session::locked_ref& session)
2245 {
2246         int ret;
2247         enum lttng_error_code status = LTTNG_OK;
2248         struct consumer_socket *socket;
2249         struct lttng_ht_iter iter;
2250         struct ltt_kernel_session *ksess = session->kernel_session;
2251
2252         LTTNG_ASSERT(ksess);
2253         LTTNG_ASSERT(ksess->consumer);
2254
2255         DBG("Clear kernel session %s (session %" PRIu64 ")", session->name, session->id);
2256
2257         if (ksess->active) {
2258                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2259                 status = LTTNG_ERR_FATAL;
2260                 goto end;
2261         }
2262
2263         {
2264                 /*
2265                  * Note that this loop will end after one iteration given that there is
2266                  * only one kernel consumer.
2267                  */
2268                 const lttng::urcu::read_lock_guard read_lock;
2269
2270                 cds_lfht_for_each_entry (
2271                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
2272                         struct ltt_kernel_channel *chan;
2273
2274                         /* For each channel, ask the consumer to clear it. */
2275                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
2276                                 DBG("Clear kernel channel %" PRIu64 ", session %s",
2277                                     chan->key,
2278                                     session->name);
2279                                 ret = consumer_clear_channel(socket, chan->key);
2280                                 if (ret < 0) {
2281                                         goto error;
2282                                 }
2283                         }
2284
2285                         if (!ksess->metadata) {
2286                                 /*
2287                                  * Nothing to do for the metadata.
2288                                  * This is a snapshot session.
2289                                  * The metadata is genererated on the fly.
2290                                  */
2291                                 continue;
2292                         }
2293
2294                         /*
2295                          * Clear the metadata channel.
2296                          * Metadata channel is not cleared per se but we still need to
2297                          * perform a rotation operation on it behind the scene.
2298                          */
2299                         ret = consumer_clear_channel(socket, ksess->metadata->key);
2300                         if (ret < 0) {
2301                                 goto error;
2302                         }
2303                 }
2304         }
2305
2306         goto end;
2307 error:
2308         switch (-ret) {
2309         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2310                 status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2311                 break;
2312         default:
2313                 status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2314                 break;
2315         }
2316 end:
2317         return status;
2318 }
2319
2320 enum lttng_error_code
2321 kernel_create_event_notifier_group_notification_fd(int *event_notifier_group_notification_fd)
2322 {
2323         int local_fd = -1, ret;
2324         enum lttng_error_code error_code_ret;
2325
2326         LTTNG_ASSERT(event_notifier_group_notification_fd);
2327
2328         ret = kernctl_create_event_notifier_group_notification_fd(
2329                 kernel_tracer_event_notifier_group_fd);
2330         if (ret < 0) {
2331                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2332                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2333                 goto error;
2334         }
2335
2336         local_fd = ret;
2337
2338         /* Prevent fd duplication after execlp(). */
2339         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2340         if (ret < 0) {
2341                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2342                        local_fd);
2343                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2344                 goto error;
2345         }
2346
2347         DBG("Created kernel notifier group notification file descriptor: fd = %d", local_fd);
2348         error_code_ret = LTTNG_OK;
2349         *event_notifier_group_notification_fd = local_fd;
2350         local_fd = -1;
2351
2352 error:
2353         if (local_fd >= 0) {
2354                 ret = close(local_fd);
2355                 if (ret) {
2356                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2357                                local_fd);
2358                 }
2359         }
2360
2361         return error_code_ret;
2362 }
2363
2364 enum lttng_error_code
2365 kernel_destroy_event_notifier_group_notification_fd(int event_notifier_group_notification_fd)
2366 {
2367         const lttng_error_code ret_code = LTTNG_OK;
2368
2369         DBG("Closing event notifier group notification file descriptor: fd = %d",
2370             event_notifier_group_notification_fd);
2371         if (event_notifier_group_notification_fd >= 0) {
2372                 const int ret = close(event_notifier_group_notification_fd);
2373                 if (ret) {
2374                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2375                                event_notifier_group_notification_fd);
2376                 }
2377         }
2378
2379         return ret_code;
2380 }
2381
2382 static unsigned long hash_trigger(const struct lttng_trigger *trigger)
2383 {
2384         const struct lttng_condition *condition = lttng_trigger_get_const_condition(trigger);
2385
2386         return lttng_condition_hash(condition);
2387 }
2388
2389 static int match_trigger(struct cds_lfht_node *node, const void *key)
2390 {
2391         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2392         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2393
2394         event_notifier_rule =
2395                 caa_container_of(node, const struct ltt_kernel_event_notifier_rule, ht_node);
2396
2397         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2398 }
2399
2400 static enum lttng_error_code kernel_create_event_notifier_rule(
2401         struct lttng_trigger *trigger, const struct lttng_credentials *creds, uint64_t token)
2402 {
2403         int err, fd, ret = 0;
2404         enum lttng_error_code error_code_ret;
2405         enum lttng_condition_status condition_status;
2406         enum lttng_condition_type condition_type;
2407         enum lttng_event_rule_type event_rule_type;
2408         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2409         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2410         unsigned int capture_bytecode_count = 0, i;
2411         const struct lttng_condition *condition = nullptr;
2412         const struct lttng_event_rule *event_rule = nullptr;
2413         enum lttng_condition_status cond_status;
2414
2415         LTTNG_ASSERT(trigger);
2416
2417         condition = lttng_trigger_get_const_condition(trigger);
2418         LTTNG_ASSERT(condition);
2419
2420         condition_type = lttng_condition_get_type(condition);
2421         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2422
2423         /* Does not acquire a reference. */
2424         condition_status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2425         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2426         LTTNG_ASSERT(event_rule);
2427
2428         event_rule_type = lttng_event_rule_get_type(event_rule);
2429         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2430
2431         error_code_ret = trace_kernel_create_event_notifier_rule(
2432                 trigger,
2433                 token,
2434                 lttng_condition_event_rule_matches_get_error_counter_index(condition),
2435                 &event_notifier_rule);
2436         if (error_code_ret != LTTNG_OK) {
2437                 goto error;
2438         }
2439
2440         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(event_rule,
2441                                                                           &kernel_event_notifier);
2442         if (error_code_ret != LTTNG_OK) {
2443                 goto free_event;
2444         }
2445
2446         kernel_event_notifier.event.token = event_notifier_rule->token;
2447         kernel_event_notifier.error_counter_idx =
2448                 lttng_condition_event_rule_matches_get_error_counter_index(condition);
2449
2450         fd = kernctl_create_event_notifier(kernel_tracer_event_notifier_group_fd,
2451                                            &kernel_event_notifier);
2452         if (fd < 0) {
2453                 switch (-fd) {
2454                 case EEXIST:
2455                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2456                         break;
2457                 case ENOSYS:
2458                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2459                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2460                         break;
2461                 case ENOENT:
2462                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2463                              kernel_event_notifier.event.name);
2464                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2465                         break;
2466                 default:
2467                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2468                                fd,
2469                                kernel_event_notifier.event.name);
2470                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2471                 }
2472                 goto free_event;
2473         }
2474
2475         event_notifier_rule->fd = fd;
2476         /* Prevent fd duplication after execlp(). */
2477         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2478         if (err < 0) {
2479                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2480                        fd);
2481                 error_code_ret = LTTNG_ERR_FATAL;
2482                 goto set_cloexec_error;
2483         }
2484
2485         if (event_notifier_rule->filter) {
2486                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2487                 if (err < 0) {
2488                         switch (-err) {
2489                         case ENOMEM:
2490                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2491                                 break;
2492                         default:
2493                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2494                                 break;
2495                         }
2496                         goto filter_error;
2497                 }
2498         }
2499
2500         if (lttng_event_rule_get_type(event_rule) == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2501                 ret = userspace_probe_event_rule_add_callsites(
2502                         event_rule, creds, event_notifier_rule->fd);
2503                 if (ret) {
2504                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2505                         goto add_callsite_error;
2506                 }
2507         }
2508
2509         /* Set the capture bytecode if any. */
2510         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2511                 condition, &capture_bytecode_count);
2512         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2513
2514         for (i = 0; i < capture_bytecode_count; i++) {
2515                 const struct lttng_bytecode *capture_bytecode =
2516                         lttng_condition_event_rule_matches_get_capture_bytecode_at_index(condition,
2517                                                                                          i);
2518
2519                 if (capture_bytecode == nullptr) {
2520                         ERR("Unexpected NULL capture bytecode on condition");
2521                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2522                         goto capture_error;
2523                 }
2524
2525                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2526                 if (ret < 0) {
2527                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2528                             event_notifier_rule->fd);
2529                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2530                         goto capture_error;
2531                 }
2532         }
2533
2534         err = kernctl_enable(event_notifier_rule->fd);
2535         if (err < 0) {
2536                 switch (-err) {
2537                 case EEXIST:
2538                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2539                         break;
2540                 default:
2541                         PERROR("enable kernel event notifier");
2542                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2543                         break;
2544                 }
2545                 goto enable_error;
2546         }
2547
2548         /* Add trigger to kernel token mapping in the hash table. */
2549         {
2550                 const lttng::urcu::read_lock_guard read_lock;
2551                 cds_lfht_add(kernel_token_to_event_notifier_rule_ht,
2552                              hash_trigger(trigger),
2553                              &event_notifier_rule->ht_node);
2554         }
2555
2556         DBG("Created kernel event notifier: name = '%s', fd = %d",
2557             kernel_event_notifier.event.name,
2558             event_notifier_rule->fd);
2559
2560         return LTTNG_OK;
2561
2562 capture_error:
2563 add_callsite_error:
2564 enable_error:
2565 set_cloexec_error:
2566 filter_error:
2567 {
2568         const int close_ret = close(event_notifier_rule->fd);
2569
2570         if (close_ret) {
2571                 PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2572                        event_notifier_rule->fd);
2573         }
2574 }
2575 free_event:
2576         free(event_notifier_rule);
2577 error:
2578         return error_code_ret;
2579 }
2580
2581 enum lttng_error_code kernel_register_event_notifier(struct lttng_trigger *trigger,
2582                                                      const struct lttng_credentials *cmd_creds)
2583 {
2584         enum lttng_error_code ret;
2585         enum lttng_condition_status status;
2586         enum lttng_domain_type domain_type;
2587         const struct lttng_event_rule *event_rule;
2588         const struct lttng_condition *const condition = lttng_trigger_get_const_condition(trigger);
2589         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2590
2591         LTTNG_ASSERT(condition);
2592
2593         /* Does not acquire a reference to the event rule. */
2594         status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2595         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2596
2597         domain_type = lttng_event_rule_get_domain_type(event_rule);
2598         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2599
2600         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2601         if (ret != LTTNG_OK) {
2602                 ERR("Failed to create kernel event notifier rule");
2603         }
2604
2605         return ret;
2606 }
2607
2608 enum lttng_error_code kernel_unregister_event_notifier(const struct lttng_trigger *trigger)
2609 {
2610         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2611         struct cds_lfht_node *node;
2612         struct cds_lfht_iter iter;
2613         enum lttng_error_code error_code_ret;
2614         int ret;
2615
2616         const lttng::urcu::read_lock_guard read_lock;
2617
2618         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2619                         hash_trigger(trigger),
2620                         match_trigger,
2621                         trigger,
2622                         &iter);
2623
2624         node = cds_lfht_iter_get_node(&iter);
2625         if (!node) {
2626                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2627                 goto error;
2628         }
2629
2630         token_event_rule_element =
2631                 caa_container_of(node, struct ltt_kernel_event_notifier_rule, ht_node);
2632
2633         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2634         if (ret) {
2635                 error_code_ret = LTTNG_ERR_FATAL;
2636                 goto error;
2637         }
2638
2639         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2640         error_code_ret = LTTNG_OK;
2641
2642 error:
2643
2644         return error_code_ret;
2645 }
2646
2647 int kernel_get_notification_fd()
2648 {
2649         return kernel_tracer_event_notifier_group_notification_fd;
2650 }