src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include "condition-internal.hpp"
  10 #include "consumer.hpp"
  11 #include "event-notifier-error-accounting.hpp"
  12 #include "kern-modules.hpp"
  13 #include "kernel-consumer.hpp"
  14 #include "kernel.hpp"
  15 #include "lttng-sessiond.hpp"
  16 #include "lttng-syscall.hpp"
  17 #include "modprobe.hpp"
  18 #include "notification-thread-commands.hpp"
  19 #include "sessiond-config.hpp"
  20 #include "tracker.hpp"
  21 #include "utils.hpp"
  22
  23 #include <common/common.hpp>
  24 #include <common/hashtable/utils.hpp>
  25 #include <common/kernel-ctl/kernel-ctl.hpp>
  26 #include <common/kernel-ctl/kernel-ioctl.hpp>
  27 #include <common/scope-exit.hpp>
  28 #include <common/sessiond-comm/sessiond-comm.hpp>
  29 #include <common/trace-chunk.hpp>
  30 #include <common/tracker.hpp>
  31 #include <common/urcu.hpp>
  32 #include <common/utils.hpp>
  33
  34 #include <lttng/condition/event-rule-matches-internal.hpp>
  35 #include <lttng/condition/event-rule-matches.h>
  36 #include <lttng/event-rule/event-rule-internal.hpp>
  37 #include <lttng/event-rule/event-rule.h>
  38 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  39 #include <lttng/event.h>
  40 #include <lttng/lttng-error.h>
  41 #include <lttng/tracker.h>
  42 #include <lttng/userspace-probe-internal.hpp>
  43 #include <lttng/userspace-probe.h>
  44
  45 #include <fcntl.h>
  46 #include <inttypes.h>
  47 #include <stdio.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <sys/types.h>
  51 #include <unistd.h>
  52
  53 namespace {
  54 /*
  55  * Key used to reference a channel between the sessiond and the consumer. This
  56  * is only read and updated with the session_list lock held.
  57  */
  58 uint64_t next_kernel_channel_key;
  59
  60 const char *module_proc_lttng = "/proc/lttng";
  61
  62 int kernel_tracer_fd = -1;
  63 nonstd::optional<enum lttng_kernel_tracer_status> kernel_tracer_status = nonstd::nullopt;
  64 int kernel_tracer_event_notifier_group_fd = -1;
  65 int kernel_tracer_event_notifier_group_notification_fd = -1;
  66 struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  67
  68 const char *kernel_tracer_status_to_str(lttng_kernel_tracer_status status)
  69 {
  70         switch (status) {
  71         case LTTNG_KERNEL_TRACER_STATUS_INITIALIZED:
  72                 return "LTTNG_KERNEL_TRACER_STATUS_INITIALIZED";
  73         case LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN:
  74                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN";
  75         case LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT:
  76                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT";
  77         case LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER:
  78                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER";
  79         case LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG:
  80                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG";
  81         case LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH:
  82                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH";
  83         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN:
  84                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN";
  85         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING:
  86                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING";
  87         case LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE:
  88                 return "LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE";
  89         }
  90
  91         abort();
  92 }
  93
  94 /*
  95  * On some architectures, calling convention details are embedded in the symbol
  96  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  97  * where an instruction boundary would be legal) to add
  98  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  99  * a per-architecture basis.
 100  */
 101 #if defined(__arm__) || defined(__aarch64__)
 102 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
 103 {
 104         /*
 105          * The least significant bit is used when branching to switch to thumb
 106          * ISA. However, it's an invalid address for us; mask the least
 107          * significant bit.
 108          */
 109         return raw_offset &= ~0b1;
 110 }
 111 #else /* defined(__arm__) || defined(__aarch64__) */
 112 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
 113 {
 114         return raw_offset;
 115 }
 116 #endif
 117 } /* namespace */
 118
 119 /*
 120  * Add context on a kernel channel.
 121  *
 122  * Assumes the ownership of ctx.
 123  */
 124 int kernel_add_channel_context(struct ltt_kernel_channel *chan, struct ltt_kernel_context *ctx)
 125 {
 126         int ret;
 127
 128         LTTNG_ASSERT(chan);
 129         LTTNG_ASSERT(ctx);
 130
 131         DBG("Adding context to channel %s", chan->channel->name);
 132         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 133         if (ret < 0) {
 134                 switch (-ret) {
 135                 case ENOSYS:
 136                         /* Exists but not available for this kernel */
 137                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 138                         goto error;
 139                 case EEXIST:
 140                         /* If EEXIST, we just ignore the error */
 141                         ret = 0;
 142                         goto end;
 143                 default:
 144                         PERROR("add context ioctl");
 145                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 146                         goto error;
 147                 }
 148         }
 149         ret = 0;
 150
 151 end:
 152         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 153         ctx->in_list = true;
 154         ctx = nullptr;
 155 error:
 156         if (ctx) {
 157                 trace_kernel_destroy_context(ctx);
 158         }
 159         return ret;
 160 }
 161
 162 /*
 163  * Create a new kernel session, register it to the kernel tracer and add it to
 164  * the session daemon session.
 165  */
 166 int kernel_create_session(struct ltt_session *session)
 167 {
 168         int ret;
 169         struct ltt_kernel_session *lks;
 170
 171         LTTNG_ASSERT(session);
 172
 173         /* Allocate data structure */
 174         lks = trace_kernel_create_session();
 175         if (lks == nullptr) {
 176                 ret = -1;
 177                 goto error;
 178         }
 179
 180         /* Kernel tracer session creation */
 181         ret = kernctl_create_session(kernel_tracer_fd);
 182         if (ret < 0) {
 183                 PERROR("ioctl kernel create session");
 184                 goto error;
 185         }
 186
 187         lks->fd = ret;
 188         /* Prevent fd duplication after execlp() */
 189         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 190         if (ret < 0) {
 191                 PERROR("fcntl session fd");
 192         }
 193
 194         lks->id = session->id;
 195         lks->consumer_fds_sent = 0;
 196         session->kernel_session = lks;
 197
 198         DBG("Kernel session created (fd: %d)", lks->fd);
 199
 200         /*
 201          * This is necessary since the creation time is present in the session
 202          * name when it is generated.
 203          */
 204         if (session->has_auto_generated_name) {
 205                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 206         } else {
 207                 ret = kernctl_session_set_name(lks->fd, session->name);
 208         }
 209         if (ret) {
 210                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 211                      session->id,
 212                      session->name);
 213         }
 214
 215         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 216         if (ret) {
 217                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 218                      session->id,
 219                      session->name);
 220         }
 221
 222         return 0;
 223
 224 error:
 225         if (lks) {
 226                 trace_kernel_destroy_session(lks);
 227                 trace_kernel_free_session(lks);
 228         }
 229         return ret;
 230 }
 231
 232 /*
 233  * Create a kernel channel, register it to the kernel tracer and add it to the
 234  * kernel session.
 235  */
 236 int kernel_create_channel(struct ltt_kernel_session *session, struct lttng_channel *chan)
 237 {
 238         int ret;
 239         struct ltt_kernel_channel *lkc;
 240
 241         LTTNG_ASSERT(session);
 242         LTTNG_ASSERT(chan);
 243
 244         /* Allocate kernel channel */
 245         lkc = trace_kernel_create_channel(chan);
 246         if (lkc == nullptr) {
 247                 goto error;
 248         }
 249
 250         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 251              chan->name,
 252              lkc->channel->attr.overwrite,
 253              lkc->channel->attr.subbuf_size,
 254              lkc->channel->attr.num_subbuf,
 255              lkc->channel->attr.switch_timer_interval,
 256              lkc->channel->attr.read_timer_interval,
 257              lkc->channel->attr.live_timer_interval,
 258              lkc->channel->attr.output);
 259
 260         /* Kernel tracer channel creation */
 261         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 262         if (ret < 0) {
 263                 PERROR("ioctl kernel create channel");
 264                 goto error;
 265         }
 266
 267         /* Setup the channel fd */
 268         lkc->fd = ret;
 269         /* Prevent fd duplication after execlp() */
 270         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 271         if (ret < 0) {
 272                 PERROR("fcntl session fd");
 273         }
 274
 275         /* Add channel to session */
 276         cds_list_add(&lkc->list, &session->channel_list.head);
 277         session->channel_count++;
 278         lkc->session = session;
 279         lkc->key = ++next_kernel_channel_key;
 280
 281         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 282             lkc->channel->name,
 283             lkc->fd,
 284             lkc->key);
 285
 286         return 0;
 287
 288 error:
 289         if (lkc) {
 290                 free(lkc->channel);
 291                 free(lkc);
 292         }
 293         return -1;
 294 }
 295
 296 /*
 297  * Create a kernel event notifier group, register it to the kernel tracer and
 298  * add it to the kernel session.
 299  */
 300 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 301 {
 302         int ret;
 303         int local_fd = -1;
 304
 305         LTTNG_ASSERT(event_notifier_group_fd);
 306
 307         /* Kernel event notifier group creation. */
 308         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 309         if (ret < 0) {
 310                 PERROR("Failed to create kernel event notifier group");
 311                 ret = -1;
 312                 goto error;
 313         }
 314
 315         local_fd = ret;
 316
 317         /* Prevent fd duplication after execlp(). */
 318         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 319         if (ret < 0) {
 320                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 321                        local_fd);
 322                 goto error;
 323         }
 324
 325         DBG("Created kernel event notifier group: fd = %d", local_fd);
 326         *event_notifier_group_fd = local_fd;
 327         local_fd = -1;
 328         ret = 0;
 329 error:
 330         if (local_fd >= 0) {
 331                 ret = close(local_fd);
 332                 if (ret) {
 333                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 334                                local_fd);
 335                 }
 336         }
 337
 338         return ret;
 339 }
 340
 341 /*
 342  * Compute the offset of the instrumentation byte in the binary based on the
 343  * function probe location using the ELF lookup method.
 344  *
 345  * Returns 0 on success and set the offset out parameter to the offset of the
 346  * elf symbol
 347  * Returns -1 on error
 348  */
 349 static int extract_userspace_probe_offset_function_elf(
 350         const struct lttng_userspace_probe_location *probe_location,
 351         uid_t uid,
 352         gid_t gid,
 353         uint64_t *offset)
 354 {
 355         int fd;
 356         int ret = 0;
 357         const char *symbol = nullptr;
 358         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 359         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 360
 361         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 362                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 363
 364         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 365         if (!lookup) {
 366                 ret = -1;
 367                 goto end;
 368         }
 369
 370         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 371
 372         LTTNG_ASSERT(lookup_method_type ==
 373                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 374
 375         symbol = lttng_userspace_probe_location_function_get_function_name(probe_location);
 376         if (!symbol) {
 377                 ret = -1;
 378                 goto end;
 379         }
 380
 381         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 382         if (fd < 0) {
 383                 ret = -1;
 384                 goto end;
 385         }
 386
 387         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 388         if (ret < 0) {
 389                 DBG("userspace probe offset calculation failed for "
 390                     "function %s",
 391                     symbol);
 392                 goto end;
 393         }
 394
 395         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t) (*offset));
 396 end:
 397         return ret;
 398 }
 399
 400 /*
 401  * Compute the offsets of the instrumentation bytes in the binary based on the
 402  * tracepoint probe location using the SDT lookup method. This function
 403  * allocates the offsets buffer, the caller must free it.
 404  *
 405  * Returns 0 on success and set the offset out parameter to the offsets of the
 406  * SDT tracepoint.
 407  * Returns -1 on error.
 408  */
 409 static int extract_userspace_probe_offset_tracepoint_sdt(
 410         const struct lttng_userspace_probe_location *probe_location,
 411         uid_t uid,
 412         gid_t gid,
 413         uint64_t **offsets,
 414         uint32_t *offsets_count)
 415 {
 416         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 417         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 418         const char *probe_name = nullptr, *provider_name = nullptr;
 419         int ret = 0;
 420         int fd, i;
 421
 422         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 423                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 424
 425         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 426         if (!lookup) {
 427                 ret = -1;
 428                 goto end;
 429         }
 430
 431         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 432
 433         LTTNG_ASSERT(lookup_method_type ==
 434                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 435
 436         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(probe_location);
 437         if (!probe_name) {
 438                 ret = -1;
 439                 goto end;
 440         }
 441
 442         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(probe_location);
 443         if (!provider_name) {
 444                 ret = -1;
 445                 goto end;
 446         }
 447
 448         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 449         if (fd < 0) {
 450                 ret = -1;
 451                 goto end;
 452         }
 453
 454         ret = run_as_extract_sdt_probe_offsets(
 455                 fd, provider_name, probe_name, uid, gid, offsets, offsets_count);
 456         if (ret < 0) {
 457                 DBG("userspace probe offset calculation failed for sdt "
 458                     "probe %s:%s",
 459                     provider_name,
 460                     probe_name);
 461                 goto end;
 462         }
 463
 464         if (*offsets_count == 0) {
 465                 DBG("no userspace probe offset found");
 466                 goto end;
 467         }
 468
 469         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 470             *offsets_count,
 471             provider_name,
 472             probe_name);
 473         for (i = 0; i < *offsets_count; i++) {
 474                 DBG("\t0x%jd", (intmax_t) ((*offsets)[i]));
 475         }
 476 end:
 477         return ret;
 478 }
 479
 480 static int userspace_probe_add_callsite(const struct lttng_userspace_probe_location *location,
 481                                         uid_t uid,
 482                                         gid_t gid,
 483                                         int fd)
 484 {
 485         const struct lttng_userspace_probe_location_lookup_method *lookup_method = nullptr;
 486         enum lttng_userspace_probe_location_lookup_method_type type;
 487         int ret;
 488
 489         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 490         if (!lookup_method) {
 491                 ret = -1;
 492                 goto end;
 493         }
 494
 495         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 496         switch (type) {
 497         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 498         {
 499                 struct lttng_kernel_abi_event_callsite callsite;
 500                 uint64_t offset;
 501
 502                 ret = extract_userspace_probe_offset_function_elf(location, uid, gid, &offset);
 503                 if (ret) {
 504                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 505                         goto end;
 506                 }
 507
 508                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 509                 ret = kernctl_add_callsite(fd, &callsite);
 510                 if (ret) {
 511                         WARN("Failed to add callsite to ELF userspace probe.");
 512                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 513                         goto end;
 514                 }
 515                 break;
 516         }
 517         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 518         {
 519                 int i;
 520                 uint64_t *offsets = nullptr;
 521                 uint32_t offsets_count;
 522                 struct lttng_kernel_abi_event_callsite callsite;
 523
 524                 /*
 525                  * This call allocates the offsets buffer. This buffer must be freed
 526                  * by the caller
 527                  */
 528                 ret = extract_userspace_probe_offset_tracepoint_sdt(
 529                         location, uid, gid, &offsets, &offsets_count);
 530                 if (ret) {
 531                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 532                         goto end;
 533                 }
 534                 for (i = 0; i < offsets_count; i++) {
 535                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 536                         ret = kernctl_add_callsite(fd, &callsite);
 537                         if (ret) {
 538                                 WARN("Failed to add callsite to SDT userspace probe");
 539                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 540                                 free(offsets);
 541                                 goto end;
 542                         }
 543                 }
 544                 free(offsets);
 545                 break;
 546         }
 547         default:
 548                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 549                 goto end;
 550         }
 551 end:
 552         return ret;
 553 }
 554
 555 /*
 556  * Extract the offsets of the instrumentation point for the different lookup
 557  * methods.
 558  */
 559 static int userspace_probe_event_add_callsites(struct lttng_event *ev,
 560                                                struct ltt_kernel_session *session,
 561                                                int fd)
 562 {
 563         int ret;
 564         const struct lttng_userspace_probe_location *location = nullptr;
 565
 566         LTTNG_ASSERT(ev);
 567         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 568
 569         location = lttng_event_get_userspace_probe_location(ev);
 570         if (!location) {
 571                 ret = -1;
 572                 goto end;
 573         }
 574
 575         ret = userspace_probe_add_callsite(location, session->uid, session->gid, fd);
 576         if (ret) {
 577                 WARN("Failed to add callsite to userspace probe event '%s'", ev->name);
 578         }
 579
 580 end:
 581         return ret;
 582 }
 583
 584 /*
 585  * Extract the offsets of the instrumentation point for the different look-up
 586  * methods.
 587  */
 588 static int userspace_probe_event_rule_add_callsites(const struct lttng_event_rule *rule,
 589                                                     const struct lttng_credentials *creds,
 590                                                     int fd)
 591 {
 592         int ret;
 593         enum lttng_event_rule_status status;
 594         enum lttng_event_rule_type event_rule_type;
 595         const struct lttng_userspace_probe_location *location = nullptr;
 596
 597         LTTNG_ASSERT(rule);
 598         LTTNG_ASSERT(creds);
 599
 600         event_rule_type = lttng_event_rule_get_type(rule);
 601         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 602
 603         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 604         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 605                 ret = -1;
 606                 goto end;
 607         }
 608
 609         ret = userspace_probe_add_callsite(
 610                 location, lttng_credentials_get_uid(creds), lttng_credentials_get_gid(creds), fd);
 611         if (ret) {
 612                 WARN("Failed to add callsite to user space probe object: fd = %d", fd);
 613         }
 614
 615 end:
 616         return ret;
 617 }
 618
 619 /*
 620  * Create a kernel event, enable it to the kernel tracer and add it to the
 621  * channel event list of the kernel session.
 622  * We own filter_expression and filter.
 623  */
 624 int kernel_create_event(struct lttng_event *ev,
 625                         struct ltt_kernel_channel *channel,
 626                         char *filter_expression,
 627                         struct lttng_bytecode *filter)
 628 {
 629         int err, fd;
 630         enum lttng_error_code ret;
 631         struct ltt_kernel_event *event;
 632
 633         LTTNG_ASSERT(ev);
 634         LTTNG_ASSERT(channel);
 635
 636         /* We pass ownership of filter_expression and filter */
 637         ret = trace_kernel_create_event(ev, filter_expression, filter, &event);
 638         if (ret != LTTNG_OK) {
 639                 goto error;
 640         }
 641
 642         fd = kernctl_create_event(channel->fd, event->event);
 643         if (fd < 0) {
 644                 switch (-fd) {
 645                 case EEXIST:
 646                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 647                         break;
 648                 case ENOSYS:
 649                         WARN("Event type not implemented");
 650                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 651                         break;
 652                 case ENOENT:
 653                         WARN("Event %s not found!", ev->name);
 654                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 655                         break;
 656                 default:
 657                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 658                         PERROR("create event ioctl");
 659                 }
 660                 goto free_event;
 661         }
 662
 663         event->type = ev->type;
 664         event->fd = fd;
 665         /* Prevent fd duplication after execlp() */
 666         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 667         if (err < 0) {
 668                 PERROR("fcntl session fd");
 669         }
 670
 671         if (filter) {
 672                 err = kernctl_filter(event->fd, filter);
 673                 if (err < 0) {
 674                         switch (-err) {
 675                         case ENOMEM:
 676                                 ret = LTTNG_ERR_FILTER_NOMEM;
 677                                 break;
 678                         default:
 679                                 ret = LTTNG_ERR_FILTER_INVAL;
 680                                 break;
 681                         }
 682                         goto filter_error;
 683                 }
 684         }
 685
 686         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 687                 ret = (lttng_error_code) userspace_probe_event_add_callsites(
 688                         ev, channel->session, event->fd);
 689                 if (ret) {
 690                         goto add_callsite_error;
 691                 }
 692         }
 693
 694         err = kernctl_enable(event->fd);
 695         if (err < 0) {
 696                 switch (-err) {
 697                 case EEXIST:
 698                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 699                         break;
 700                 default:
 701                         PERROR("enable kernel event");
 702                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 703                         break;
 704                 }
 705                 goto enable_error;
 706         }
 707
 708         /* Add event to event list */
 709         cds_list_add(&event->list, &channel->events_list.head);
 710         channel->event_count++;
 711
 712         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 713
 714         return 0;
 715
 716 add_callsite_error:
 717 enable_error:
 718 filter_error:
 719 {
 720         int closeret;
 721
 722         closeret = close(event->fd);
 723         if (closeret) {
 724                 PERROR("close event fd");
 725         }
 726 }
 727 free_event:
 728         free(event);
 729 error:
 730         return ret;
 731 }
 732
 733 /*
 734  * Disable a kernel channel.
 735  */
 736 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 737 {
 738         int ret;
 739
 740         LTTNG_ASSERT(chan);
 741
 742         ret = kernctl_disable(chan->fd);
 743         if (ret < 0) {
 744                 PERROR("disable chan ioctl");
 745                 goto error;
 746         }
 747
 748         chan->enabled = false;
 749         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 750             chan->channel->name,
 751             chan->fd,
 752             chan->key);
 753
 754         return 0;
 755
 756 error:
 757         return ret;
 758 }
 759
 760 /*
 761  * Enable a kernel channel.
 762  */
 763 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 764 {
 765         int ret;
 766
 767         LTTNG_ASSERT(chan);
 768
 769         ret = kernctl_enable(chan->fd);
 770         if (ret < 0 && ret != -EEXIST) {
 771                 PERROR("Enable kernel chan");
 772                 goto error;
 773         }
 774
 775         chan->enabled = true;
 776         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 777             chan->channel->name,
 778             chan->fd,
 779             chan->key);
 780
 781         return 0;
 782
 783 error:
 784         return ret;
 785 }
 786
 787 /*
 788  * Enable a kernel event.
 789  */
 790 int kernel_enable_event(struct ltt_kernel_event *event)
 791 {
 792         int ret;
 793
 794         LTTNG_ASSERT(event);
 795
 796         ret = kernctl_enable(event->fd);
 797         if (ret < 0) {
 798                 switch (-ret) {
 799                 case EEXIST:
 800                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 801                         break;
 802                 default:
 803                         PERROR("enable kernel event");
 804                         break;
 805                 }
 806                 goto error;
 807         }
 808
 809         event->enabled = true;
 810         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 811
 812         return 0;
 813
 814 error:
 815         return ret;
 816 }
 817
 818 /*
 819  * Disable a kernel event.
 820  */
 821 int kernel_disable_event(struct ltt_kernel_event *event)
 822 {
 823         int ret;
 824
 825         LTTNG_ASSERT(event);
 826
 827         ret = kernctl_disable(event->fd);
 828         if (ret < 0) {
 829                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 830                        event->event->name,
 831                        event->fd);
 832                 goto error;
 833         }
 834
 835         event->enabled = false;
 836         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 837
 838         return 0;
 839
 840 error:
 841         return ret;
 842 }
 843
 844 /*
 845  * Disable a kernel event notifier.
 846  */
 847 static int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 848 {
 849         int ret;
 850
 851         LTTNG_ASSERT(event);
 852
 853         lttng::urcu::read_lock_guard read_lock;
 854         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 855
 856         ret = kernctl_disable(event->fd);
 857         if (ret < 0) {
 858                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 859                        event->fd,
 860                        event->token);
 861                 goto error;
 862         }
 863
 864         event->enabled = false;
 865         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64, event->fd, event->token);
 866
 867 error:
 868         return ret;
 869 }
 870
 871 static struct process_attr_tracker *
 872 _kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 873                                  enum lttng_process_attr process_attr)
 874 {
 875         switch (process_attr) {
 876         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 877                 return session->tracker_pid;
 878         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 879                 return session->tracker_vpid;
 880         case LTTNG_PROCESS_ATTR_USER_ID:
 881                 return session->tracker_uid;
 882         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 883                 return session->tracker_vuid;
 884         case LTTNG_PROCESS_ATTR_GROUP_ID:
 885                 return session->tracker_gid;
 886         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 887                 return session->tracker_vgid;
 888         default:
 889                 return nullptr;
 890         }
 891 }
 892
 893 const struct process_attr_tracker *
 894 kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 895                                 enum lttng_process_attr process_attr)
 896 {
 897         return (const struct process_attr_tracker *) _kernel_get_process_attr_tracker(session,
 898                                                                                       process_attr);
 899 }
 900
 901 enum lttng_error_code
 902 kernel_process_attr_tracker_set_tracking_policy(struct ltt_kernel_session *session,
 903                                                 enum lttng_process_attr process_attr,
 904                                                 enum lttng_tracking_policy policy)
 905 {
 906         int ret;
 907         enum lttng_error_code ret_code = LTTNG_OK;
 908         struct process_attr_tracker *tracker =
 909                 _kernel_get_process_attr_tracker(session, process_attr);
 910         enum lttng_tracking_policy previous_policy;
 911
 912         if (!tracker) {
 913                 ret_code = LTTNG_ERR_INVALID;
 914                 goto end;
 915         }
 916
 917         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 918         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 919         if (ret) {
 920                 ret_code = LTTNG_ERR_UNK;
 921                 goto end;
 922         }
 923
 924         if (previous_policy == policy) {
 925                 goto end;
 926         }
 927
 928         switch (policy) {
 929         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 930                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 931                         /*
 932                          * Maintain a special case for the process ID process
 933                          * attribute tracker as it was the only supported
 934                          * attribute prior to 2.12.
 935                          */
 936                         ret = kernctl_track_pid(session->fd, -1);
 937                 } else {
 938                         ret = kernctl_track_id(session->fd, process_attr, -1);
 939                 }
 940                 break;
 941         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 942         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 943                 /* fall-through. */
 944                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 945                         /*
 946                          * Maintain a special case for the process ID process
 947                          * attribute tracker as it was the only supported
 948                          * attribute prior to 2.12.
 949                          */
 950                         ret = kernctl_untrack_pid(session->fd, -1);
 951                 } else {
 952                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 953                 }
 954                 break;
 955         default:
 956                 abort();
 957         }
 958         /* kern-ctl error handling */
 959         switch (-ret) {
 960         case 0:
 961                 ret_code = LTTNG_OK;
 962                 break;
 963         case EINVAL:
 964                 ret_code = LTTNG_ERR_INVALID;
 965                 break;
 966         case ENOMEM:
 967                 ret_code = LTTNG_ERR_NOMEM;
 968                 break;
 969         case EEXIST:
 970                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 971                 break;
 972         default:
 973                 ret_code = LTTNG_ERR_UNK;
 974                 break;
 975         }
 976 end:
 977         return ret_code;
 978 }
 979
 980 enum lttng_error_code
 981 kernel_process_attr_tracker_inclusion_set_add_value(struct ltt_kernel_session *session,
 982                                                     enum lttng_process_attr process_attr,
 983                                                     const struct process_attr_value *value)
 984 {
 985         int ret, integral_value;
 986         enum lttng_error_code ret_code;
 987         struct process_attr_tracker *tracker;
 988         enum process_attr_tracker_status status;
 989
 990         /*
 991          * Convert process attribute tracker value to the integral
 992          * representation required by the kern-ctl API.
 993          */
 994         switch (process_attr) {
 995         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 996         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 997                 integral_value = (int) value->value.pid;
 998                 break;
 999         case LTTNG_PROCESS_ATTR_USER_ID:
1000         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1001                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1002                         uid_t uid;
1003
1004                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1005                         if (ret_code != LTTNG_OK) {
1006                                 goto end;
1007                         }
1008                         integral_value = (int) uid;
1009                 } else {
1010                         integral_value = (int) value->value.uid;
1011                 }
1012                 break;
1013         case LTTNG_PROCESS_ATTR_GROUP_ID:
1014         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1015                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1016                         gid_t gid;
1017
1018                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1019                         if (ret_code != LTTNG_OK) {
1020                                 goto end;
1021                         }
1022                         integral_value = (int) gid;
1023                 } else {
1024                         integral_value = (int) value->value.gid;
1025                 }
1026                 break;
1027         default:
1028                 ret_code = LTTNG_ERR_INVALID;
1029                 goto end;
1030         }
1031
1032         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1033         if (!tracker) {
1034                 ret_code = LTTNG_ERR_INVALID;
1035                 goto end;
1036         }
1037
1038         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1039         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1040                 switch (status) {
1041                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1042                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1043                         break;
1044                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1045                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1046                         break;
1047                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1048                 default:
1049                         ret_code = LTTNG_ERR_UNK;
1050                         break;
1051                 }
1052                 goto end;
1053         }
1054
1055         DBG("Kernel track %s %d for session id %" PRIu64,
1056             lttng_process_attr_to_string(process_attr),
1057             integral_value,
1058             session->id);
1059         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1060                 /*
1061                  * Maintain a special case for the process ID process attribute
1062                  * tracker as it was the only supported attribute prior to 2.12.
1063                  */
1064                 ret = kernctl_track_pid(session->fd, integral_value);
1065         } else {
1066                 ret = kernctl_track_id(session->fd, process_attr, integral_value);
1067         }
1068         if (ret == 0) {
1069                 ret_code = LTTNG_OK;
1070                 goto end;
1071         }
1072
1073         kernel_wait_quiescent();
1074
1075         /* kern-ctl error handling */
1076         switch (-ret) {
1077         case 0:
1078                 ret_code = LTTNG_OK;
1079                 break;
1080         case EINVAL:
1081                 ret_code = LTTNG_ERR_INVALID;
1082                 break;
1083         case ENOMEM:
1084                 ret_code = LTTNG_ERR_NOMEM;
1085                 break;
1086         case EEXIST:
1087                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1088                 break;
1089         default:
1090                 ret_code = LTTNG_ERR_UNK;
1091                 break;
1092         }
1093
1094         /* Attempt to remove the value from the tracker. */
1095         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1096         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1097                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1098                     lttng_process_attr_to_string(process_attr),
1099                     integral_value);
1100         }
1101 end:
1102         return ret_code;
1103 }
1104
1105 enum lttng_error_code
1106 kernel_process_attr_tracker_inclusion_set_remove_value(struct ltt_kernel_session *session,
1107                                                        enum lttng_process_attr process_attr,
1108                                                        const struct process_attr_value *value)
1109 {
1110         int ret, integral_value;
1111         enum lttng_error_code ret_code;
1112         struct process_attr_tracker *tracker;
1113         enum process_attr_tracker_status status;
1114
1115         /*
1116          * Convert process attribute tracker value to the integral
1117          * representation required by the kern-ctl API.
1118          */
1119         switch (process_attr) {
1120         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1121         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1122                 integral_value = (int) value->value.pid;
1123                 break;
1124         case LTTNG_PROCESS_ATTR_USER_ID:
1125         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1126                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1127                         uid_t uid;
1128
1129                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1130                         if (ret_code != LTTNG_OK) {
1131                                 goto end;
1132                         }
1133                         integral_value = (int) uid;
1134                 } else {
1135                         integral_value = (int) value->value.uid;
1136                 }
1137                 break;
1138         case LTTNG_PROCESS_ATTR_GROUP_ID:
1139         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1140                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1141                         gid_t gid;
1142
1143                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1144                         if (ret_code != LTTNG_OK) {
1145                                 goto end;
1146                         }
1147                         integral_value = (int) gid;
1148                 } else {
1149                         integral_value = (int) value->value.gid;
1150                 }
1151                 break;
1152         default:
1153                 ret_code = LTTNG_ERR_INVALID;
1154                 goto end;
1155         }
1156
1157         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1158         if (!tracker) {
1159                 ret_code = LTTNG_ERR_INVALID;
1160                 goto end;
1161         }
1162
1163         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1164         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1165                 switch (status) {
1166                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1167                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1168                         break;
1169                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1170                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1171                         break;
1172                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1173                 default:
1174                         ret_code = LTTNG_ERR_UNK;
1175                         break;
1176                 }
1177                 goto end;
1178         }
1179
1180         DBG("Kernel track %s %d for session id %" PRIu64,
1181             lttng_process_attr_to_string(process_attr),
1182             integral_value,
1183             session->id);
1184         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1185                 /*
1186                  * Maintain a special case for the process ID process attribute
1187                  * tracker as it was the only supported attribute prior to 2.12.
1188                  */
1189                 ret = kernctl_untrack_pid(session->fd, integral_value);
1190         } else {
1191                 ret = kernctl_untrack_id(session->fd, process_attr, integral_value);
1192         }
1193         if (ret == 0) {
1194                 ret_code = LTTNG_OK;
1195                 goto end;
1196         }
1197         kernel_wait_quiescent();
1198
1199         /* kern-ctl error handling */
1200         switch (-ret) {
1201         case 0:
1202                 ret_code = LTTNG_OK;
1203                 break;
1204         case EINVAL:
1205                 ret_code = LTTNG_ERR_INVALID;
1206                 break;
1207         case ENOMEM:
1208                 ret_code = LTTNG_ERR_NOMEM;
1209                 break;
1210         case ENOENT:
1211                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1212                 break;
1213         default:
1214                 ret_code = LTTNG_ERR_UNK;
1215                 break;
1216         }
1217
1218         /* Attempt to add the value to the tracker. */
1219         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1220         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1221                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1222                     lttng_process_attr_to_string(process_attr),
1223                     integral_value);
1224         }
1225 end:
1226         return ret_code;
1227 }
1228
1229 /*
1230  * Create kernel metadata, open from the kernel tracer and add it to the
1231  * kernel session.
1232  */
1233 int kernel_open_metadata(struct ltt_kernel_session *session)
1234 {
1235         int ret;
1236         struct ltt_kernel_metadata *lkm = nullptr;
1237
1238         LTTNG_ASSERT(session);
1239
1240         /* Allocate kernel metadata */
1241         lkm = trace_kernel_create_metadata();
1242         if (lkm == nullptr) {
1243                 goto error;
1244         }
1245
1246         /* Kernel tracer metadata creation */
1247         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1248         if (ret < 0) {
1249                 goto error_open;
1250         }
1251
1252         lkm->fd = ret;
1253         lkm->key = ++next_kernel_channel_key;
1254         /* Prevent fd duplication after execlp() */
1255         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1256         if (ret < 0) {
1257                 PERROR("fcntl session fd");
1258         }
1259
1260         session->metadata = lkm;
1261
1262         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1263
1264         return 0;
1265
1266 error_open:
1267         trace_kernel_destroy_metadata(lkm);
1268 error:
1269         return -1;
1270 }
1271
1272 /*
1273  * Start tracing session.
1274  */
1275 int kernel_start_session(struct ltt_kernel_session *session)
1276 {
1277         int ret;
1278
1279         LTTNG_ASSERT(session);
1280
1281         ret = kernctl_start_session(session->fd);
1282         if (ret < 0) {
1283                 PERROR("ioctl start session");
1284                 goto error;
1285         }
1286
1287         DBG("Kernel session started");
1288
1289         return 0;
1290
1291 error:
1292         return ret;
1293 }
1294
1295 /*
1296  * Make a kernel wait to make sure in-flight probe have completed.
1297  */
1298 void kernel_wait_quiescent()
1299 {
1300         int ret;
1301         int fd = kernel_tracer_fd;
1302
1303         DBG("Kernel quiescent wait on %d", fd);
1304
1305         ret = kernctl_wait_quiescent(fd);
1306         if (ret < 0) {
1307                 PERROR("wait quiescent ioctl");
1308                 ERR("Kernel quiescent wait failed");
1309         }
1310 }
1311
1312 /*
1313  *  Force flush buffer of metadata.
1314  */
1315 int kernel_metadata_flush_buffer(int fd)
1316 {
1317         int ret;
1318
1319         DBG("Kernel flushing metadata buffer on fd %d", fd);
1320
1321         ret = kernctl_buffer_flush(fd);
1322         if (ret < 0) {
1323                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1324         }
1325
1326         return 0;
1327 }
1328
1329 /*
1330  * Force flush buffer for channel.
1331  */
1332 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1333 {
1334         int ret;
1335         struct ltt_kernel_stream *stream;
1336
1337         LTTNG_ASSERT(channel);
1338
1339         DBG("Flush buffer for channel %s", channel->channel->name);
1340
1341         cds_list_for_each_entry (stream, &channel->stream_list.head, list) {
1342                 DBG("Flushing channel stream %d", stream->fd);
1343                 ret = kernctl_buffer_flush(stream->fd);
1344                 if (ret < 0) {
1345                         PERROR("ioctl");
1346                         ERR("Fail to flush buffer for stream %d (ret: %d)", stream->fd, ret);
1347                 }
1348         }
1349
1350         return 0;
1351 }
1352
1353 /*
1354  * Stop tracing session.
1355  */
1356 int kernel_stop_session(struct ltt_kernel_session *session)
1357 {
1358         int ret;
1359
1360         LTTNG_ASSERT(session);
1361
1362         ret = kernctl_stop_session(session->fd);
1363         if (ret < 0) {
1364                 goto error;
1365         }
1366
1367         DBG("Kernel session stopped");
1368
1369         return 0;
1370
1371 error:
1372         return ret;
1373 }
1374
1375 /*
1376  * Open stream of channel, register it to the kernel tracer and add it
1377  * to the stream list of the channel.
1378  *
1379  * Note: given that the streams may appear in random order wrt CPU
1380  * number (e.g. cpu hotplug), the index value of the stream number in
1381  * the stream name is not necessarily linked to the CPU number.
1382  *
1383  * Return the number of created stream. Else, a negative value.
1384  */
1385 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1386 {
1387         int ret;
1388         struct ltt_kernel_stream *lks;
1389
1390         LTTNG_ASSERT(channel);
1391
1392         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1393                 lks = trace_kernel_create_stream(channel->channel->name, channel->stream_count);
1394                 if (lks == nullptr) {
1395                         ret = close(ret);
1396                         if (ret) {
1397                                 PERROR("close");
1398                         }
1399                         goto error;
1400                 }
1401
1402                 lks->fd = ret;
1403                 /* Prevent fd duplication after execlp() */
1404                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1405                 if (ret < 0) {
1406                         PERROR("fcntl session fd");
1407                 }
1408
1409                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1410                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1411
1412                 /* Add stream to channel stream list */
1413                 cds_list_add(&lks->list, &channel->stream_list.head);
1414                 channel->stream_count++;
1415
1416                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd, lks->state);
1417         }
1418
1419         return channel->stream_count;
1420
1421 error:
1422         return -1;
1423 }
1424
1425 /*
1426  * Open the metadata stream and set it to the kernel session.
1427  */
1428 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1429 {
1430         int ret;
1431
1432         LTTNG_ASSERT(session);
1433
1434         ret = kernctl_create_stream(session->metadata->fd);
1435         if (ret < 0) {
1436                 PERROR("kernel create metadata stream");
1437                 goto error;
1438         }
1439
1440         DBG("Kernel metadata stream created (fd: %d)", ret);
1441         session->metadata_stream_fd = ret;
1442         /* Prevent fd duplication after execlp() */
1443         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1444         if (ret < 0) {
1445                 PERROR("fcntl session fd");
1446         }
1447
1448         return 0;
1449
1450 error:
1451         return -1;
1452 }
1453
1454 /*
1455  * Get the event list from the kernel tracer and return the number of elements.
1456  */
1457 ssize_t kernel_list_events(struct lttng_event **events)
1458 {
1459         int fd, ret;
1460         char *event;
1461         size_t nbmem, count = 0;
1462         FILE *fp;
1463         struct lttng_event *elist;
1464
1465         LTTNG_ASSERT(events);
1466
1467         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1468         if (fd < 0) {
1469                 PERROR("kernel tracepoint list");
1470                 goto error;
1471         }
1472
1473         fp = fdopen(fd, "r");
1474         if (fp == nullptr) {
1475                 PERROR("kernel tracepoint list fdopen");
1476                 goto error_fp;
1477         }
1478
1479         /*
1480          * Init memory size counter
1481          * See kernel-ctl.h for explanation of this value
1482          */
1483         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1484         elist = calloc<lttng_event>(nbmem);
1485         if (elist == nullptr) {
1486                 PERROR("alloc list events");
1487                 count = -ENOMEM;
1488                 goto end;
1489         }
1490
1491         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1492                 if (count >= nbmem) {
1493                         struct lttng_event *new_elist;
1494                         size_t new_nbmem;
1495
1496                         new_nbmem = nbmem << 1;
1497                         DBG("Reallocating event list from %zu to %zu bytes", nbmem, new_nbmem);
1498                         new_elist = (lttng_event *) realloc(elist,
1499                                                             new_nbmem * sizeof(struct lttng_event));
1500                         if (new_elist == nullptr) {
1501                                 PERROR("realloc list events");
1502                                 free(event);
1503                                 free(elist);
1504                                 count = -ENOMEM;
1505                                 goto end;
1506                         }
1507                         /* Zero the new memory */
1508                         memset(new_elist + nbmem,
1509                                0,
1510                                (new_nbmem - nbmem) * sizeof(struct lttng_event));
1511                         nbmem = new_nbmem;
1512                         elist = new_elist;
1513                 }
1514                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1515                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1516                 elist[count].enabled = -1;
1517                 count++;
1518                 free(event);
1519         }
1520
1521         *events = elist;
1522         DBG("Kernel list events done (%zu events)", count);
1523 end:
1524         ret = fclose(fp); /* closes both fp and fd */
1525         if (ret) {
1526                 PERROR("fclose");
1527         }
1528         return count;
1529
1530 error_fp:
1531         ret = close(fd);
1532         if (ret) {
1533                 PERROR("close");
1534         }
1535 error:
1536         return -1;
1537 }
1538
1539 /*
1540  * Get kernel version and validate it.
1541  */
1542 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1543                             struct lttng_kernel_abi_tracer_abi_version *abi_version)
1544 {
1545         int ret;
1546
1547         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1548         if (ret < 0) {
1549                 ERR("Failed to retrieve the lttng-modules version");
1550                 goto error;
1551         }
1552
1553         /* Validate version */
1554         if (version->major != VERSION_MAJOR) {
1555                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1556                     version->major,
1557                     VERSION_MAJOR);
1558                 goto error_version;
1559         }
1560         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1561         if (ret < 0) {
1562                 ERR("Failed to retrieve lttng-modules ABI version");
1563                 goto error;
1564         }
1565         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1566                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1567                     abi_version->major,
1568                     abi_version->minor,
1569                     LTTNG_KERNEL_ABI_MAJOR_VERSION);
1570                 goto error;
1571         }
1572         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1573              version->major,
1574              version->minor,
1575              abi_version->major,
1576              abi_version->minor);
1577         return 0;
1578
1579 error_version:
1580         ret = -1;
1581
1582 error:
1583         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1584         return ret;
1585 }
1586
1587 /*
1588  * Kernel work-arounds called at the start of sessiond main().
1589  */
1590 int init_kernel_workarounds()
1591 {
1592         int ret;
1593         FILE *fp;
1594
1595         /*
1596          * boot_id needs to be read once before being used concurrently
1597          * to deal with a Linux kernel race. A fix is proposed for
1598          * upstream, but the work-around is needed for older kernels.
1599          */
1600         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1601         if (!fp) {
1602                 goto end_boot_id;
1603         }
1604         while (!feof(fp)) {
1605                 char buf[37] = "";
1606
1607                 ret = fread(buf, 1, sizeof(buf), fp);
1608                 if (ret < 0) {
1609                         /* Ignore error, we don't really care */
1610                 }
1611         }
1612         ret = fclose(fp);
1613         if (ret) {
1614                 PERROR("fclose");
1615         }
1616 end_boot_id:
1617         return 0;
1618 }
1619
1620 /*
1621  * Teardown of a kernel session, keeping data required by destroy notifiers.
1622  */
1623 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1624 {
1625         struct lttng_trace_chunk *trace_chunk;
1626
1627         if (ksess == nullptr) {
1628                 DBG3("No kernel session when tearing down session");
1629                 return;
1630         }
1631
1632         DBG("Tearing down kernel session");
1633         trace_chunk = ksess->current_trace_chunk;
1634
1635         /*
1636          * Destroy channels on the consumer if at least one FD has been sent and we
1637          * are in no output mode because the streams are in *no* monitor mode so we
1638          * have to send a command to clean them up or else they leaked.
1639          */
1640         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1641                 int ret;
1642                 struct consumer_socket *socket;
1643                 struct lttng_ht_iter iter;
1644
1645                 /* For each consumer socket. */
1646                 lttng::urcu::read_lock_guard read_lock;
1647
1648                 cds_lfht_for_each_entry (
1649                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1650                         struct ltt_kernel_channel *chan;
1651
1652                         /* For each channel, ask the consumer to destroy it. */
1653                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1654                                 ret = kernel_consumer_destroy_channel(socket, chan);
1655                                 if (ret < 0) {
1656                                         /* Consumer is probably dead. Use next socket. */
1657                                         continue;
1658                                 }
1659                         }
1660                 }
1661         }
1662
1663         /* Close any relayd session */
1664         consumer_output_send_destroy_relayd(ksess->consumer);
1665
1666         trace_kernel_destroy_session(ksess);
1667         lttng_trace_chunk_put(trace_chunk);
1668 }
1669
1670 /* Teardown of data required by destroy notifiers. */
1671 void kernel_free_session(struct ltt_kernel_session *ksess)
1672 {
1673         if (ksess == nullptr) {
1674                 return;
1675         }
1676         trace_kernel_free_session(ksess);
1677 }
1678
1679 /*
1680  * Destroy a kernel channel object. It does not do anything on the tracer side.
1681  */
1682 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1683 {
1684         struct ltt_kernel_session *ksess = nullptr;
1685
1686         LTTNG_ASSERT(kchan);
1687         LTTNG_ASSERT(kchan->channel);
1688
1689         DBG3("Kernel destroy channel %s", kchan->channel->name);
1690
1691         /* Update channel count of associated session. */
1692         if (kchan->session) {
1693                 /* Keep pointer reference so we can update it after the destroy. */
1694                 ksess = kchan->session;
1695         }
1696
1697         trace_kernel_destroy_channel(kchan);
1698
1699         /*
1700          * At this point the kernel channel is not visible anymore. This is safe
1701          * since in order to work on a visible kernel session, the tracing session
1702          * lock (ltt_session.lock) MUST be acquired.
1703          */
1704         if (ksess) {
1705                 ksess->channel_count--;
1706         }
1707 }
1708
1709 /*
1710  * Take a snapshot for a given kernel session.
1711  *
1712  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1713  */
1714 enum lttng_error_code kernel_snapshot_record(struct ltt_kernel_session *ksess,
1715                                              const struct consumer_output *output,
1716                                              uint64_t nb_packets_per_stream)
1717 {
1718         int err, ret, saved_metadata_fd;
1719         enum lttng_error_code status = LTTNG_OK;
1720         struct consumer_socket *socket;
1721         struct lttng_ht_iter iter;
1722         struct ltt_kernel_metadata *saved_metadata;
1723         char *trace_path = nullptr;
1724         size_t consumer_path_offset = 0;
1725
1726         LTTNG_ASSERT(ksess);
1727         LTTNG_ASSERT(ksess->consumer);
1728         LTTNG_ASSERT(output);
1729
1730         DBG("Kernel snapshot record started");
1731
1732         /* Save current metadata since the following calls will change it. */
1733         saved_metadata = ksess->metadata;
1734         saved_metadata_fd = ksess->metadata_stream_fd;
1735
1736         ret = kernel_open_metadata(ksess);
1737         if (ret < 0) {
1738                 status = LTTNG_ERR_KERN_META_FAIL;
1739                 goto error;
1740         }
1741
1742         ret = kernel_open_metadata_stream(ksess);
1743         if (ret < 0) {
1744                 status = LTTNG_ERR_KERN_META_FAIL;
1745                 goto error_open_stream;
1746         }
1747
1748         trace_path = setup_channel_trace_path(ksess->consumer, "", &consumer_path_offset);
1749         if (!trace_path) {
1750                 status = LTTNG_ERR_INVALID;
1751                 goto error;
1752         }
1753
1754         {
1755                 /* Send metadata to consumer and snapshot everything. */
1756                 lttng::urcu::read_lock_guard read_lock;
1757
1758                 cds_lfht_for_each_entry (output->socks->ht, &iter.iter, socket, node.node) {
1759                         struct ltt_kernel_channel *chan;
1760
1761                         pthread_mutex_lock(socket->lock);
1762                         /* This stream must not be monitored by the consumer. */
1763                         ret = kernel_consumer_add_metadata(socket, ksess, 0);
1764                         pthread_mutex_unlock(socket->lock);
1765                         if (ret < 0) {
1766                                 status = LTTNG_ERR_KERN_META_FAIL;
1767                                 goto error_consumer;
1768                         }
1769
1770                         /* For each channel, ask the consumer to snapshot it. */
1771                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1772                                 status =
1773                                         consumer_snapshot_channel(socket,
1774                                                                   chan->key,
1775                                                                   output,
1776                                                                   0,
1777                                                                   &trace_path[consumer_path_offset],
1778                                                                   nb_packets_per_stream);
1779                                 if (status != LTTNG_OK) {
1780                                         (void) kernel_consumer_destroy_metadata(socket,
1781                                                                                 ksess->metadata);
1782                                         goto error_consumer;
1783                                 }
1784                         }
1785
1786                         /* Snapshot metadata, */
1787                         status = consumer_snapshot_channel(socket,
1788                                                            ksess->metadata->key,
1789                                                            output,
1790                                                            1,
1791                                                            &trace_path[consumer_path_offset],
1792                                                            0);
1793                         if (status != LTTNG_OK) {
1794                                 goto error_consumer;
1795                         }
1796
1797                         /*
1798                          * The metadata snapshot is done, ask the consumer to destroy it since
1799                          * it's not monitored on the consumer side.
1800                          */
1801                         (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1802                 }
1803         }
1804
1805 error_consumer:
1806         /* Close newly opened metadata stream. It's now on the consumer side. */
1807         err = close(ksess->metadata_stream_fd);
1808         if (err < 0) {
1809                 PERROR("close snapshot kernel");
1810         }
1811
1812 error_open_stream:
1813         trace_kernel_destroy_metadata(ksess->metadata);
1814 error:
1815         /* Restore metadata state.*/
1816         ksess->metadata = saved_metadata;
1817         ksess->metadata_stream_fd = saved_metadata_fd;
1818         free(trace_path);
1819         return status;
1820 }
1821
1822 /*
1823  * Get the syscall mask array from the kernel tracer.
1824  *
1825  * Return 0 on success else a negative value. In both case, syscall_mask should
1826  * be freed.
1827  */
1828 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1829 {
1830         LTTNG_ASSERT(syscall_mask);
1831         LTTNG_ASSERT(nr_bits);
1832
1833         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1834 }
1835
1836 static int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1837 {
1838         int ret;
1839         struct lttng_kernel_abi_tracer_abi_version abi;
1840
1841         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1842         if (ret < 0) {
1843                 ERR("Failed to retrieve lttng-modules ABI version");
1844                 goto error;
1845         }
1846
1847         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1848 error:
1849         return ret;
1850 }
1851
1852 /*
1853  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1854  * version number.
1855  *
1856  * Return 1 on success, 0 when feature is not supported, negative value in case
1857  * of errors.
1858  */
1859 int kernel_supports_ring_buffer_snapshot_sample_positions()
1860 {
1861         /*
1862          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1863          */
1864         return kernel_tracer_abi_greater_or_equal(2, 3);
1865 }
1866
1867 /*
1868  * Check for the support of the packet sequence number via abi version number.
1869  *
1870  * Return 1 on success, 0 when feature is not supported, negative value in case
1871  * of errors.
1872  */
1873 int kernel_supports_ring_buffer_packet_sequence_number()
1874 {
1875         /*
1876          * Packet sequence number was introduced in LTTng 2.8,
1877          * lttng-modules ABI 2.1.
1878          */
1879         return kernel_tracer_abi_greater_or_equal(2, 1);
1880 }
1881
1882 /*
1883  * Check for the support of event notifiers via abi version number.
1884  *
1885  * Return 1 on success, 0 when feature is not supported, negative value in case
1886  * of errors.
1887  */
1888 int kernel_supports_event_notifiers()
1889 {
1890         /*
1891          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1892          */
1893         return kernel_tracer_abi_greater_or_equal(2, 6);
1894 }
1895
1896 /*
1897  * Rotate a kernel session.
1898  *
1899  * Return LTTNG_OK on success or else an LTTng error code.
1900  */
1901 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1902 {
1903         int ret;
1904         enum lttng_error_code status = LTTNG_OK;
1905         struct consumer_socket *socket;
1906         struct lttng_ht_iter iter;
1907         struct ltt_kernel_session *ksess = session->kernel_session;
1908
1909         LTTNG_ASSERT(ksess);
1910         LTTNG_ASSERT(ksess->consumer);
1911
1912         DBG("Rotate kernel session %s started (session %" PRIu64 ")", session->name, session->id);
1913
1914         {
1915                 /*
1916                  * Note that this loop will end after one iteration given that there is
1917                  * only one kernel consumer.
1918                  */
1919                 lttng::urcu::read_lock_guard read_lock;
1920
1921                 cds_lfht_for_each_entry (
1922                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1923                         struct ltt_kernel_channel *chan;
1924
1925                         /* For each channel, ask the consumer to rotate it. */
1926                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1927                                 DBG("Rotate kernel channel %" PRIu64 ", session %s",
1928                                     chan->key,
1929                                     session->name);
1930                                 ret = consumer_rotate_channel(socket,
1931                                                               chan->key,
1932                                                               ksess->consumer,
1933                                                               /* is_metadata_channel */ false);
1934                                 if (ret < 0) {
1935                                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1936                                         goto error;
1937                                 }
1938                         }
1939
1940                         /*
1941                          * Rotate the metadata channel.
1942                          */
1943                         ret = consumer_rotate_channel(socket,
1944                                                       ksess->metadata->key,
1945                                                       ksess->consumer,
1946                                                       /* is_metadata_channel */ true);
1947                         if (ret < 0) {
1948                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1949                                 goto error;
1950                         }
1951                 }
1952         }
1953
1954 error:
1955         return status;
1956 }
1957
1958 enum lttng_error_code kernel_create_channel_subdirectories(const struct ltt_kernel_session *ksess)
1959 {
1960         enum lttng_error_code ret = LTTNG_OK;
1961         enum lttng_trace_chunk_status chunk_status;
1962
1963         lttng::urcu::read_lock_guard read_lock;
1964         LTTNG_ASSERT(ksess->current_trace_chunk);
1965
1966         /*
1967          * Create the index subdirectory which will take care
1968          * of implicitly creating the channel's path.
1969          */
1970         chunk_status = lttng_trace_chunk_create_subdirectory(
1971                 ksess->current_trace_chunk, DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1972         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1973                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1974                 goto error;
1975         }
1976 error:
1977         return ret;
1978 }
1979
1980 /*
1981  * Get current kernel tracer status
1982  */
1983 enum lttng_kernel_tracer_status get_kernel_tracer_status()
1984 {
1985         if (!kernel_tracer_status) {
1986                 return LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN;
1987         }
1988
1989         return *kernel_tracer_status;
1990 }
1991
1992 /*
1993  * Sets the kernel tracer status based on the positive errno code
1994  */
1995 void set_kernel_tracer_status_from_modules_ret(int code)
1996 {
1997         switch (code) {
1998         case ENOENT:
1999         {
2000                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2001                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING);
2002                 break;
2003         }
2004         case ENOKEY:
2005         case EKEYEXPIRED:
2006         case EKEYREVOKED:
2007         case EKEYREJECTED:
2008         {
2009                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2010                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE);
2011                 break;
2012         }
2013         default:
2014         {
2015                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2016                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN);
2017                 break;
2018         }
2019         }
2020 }
2021
2022 /*
2023  * Setup necessary data for kernel tracer action.
2024  */
2025 int init_kernel_tracer()
2026 {
2027         int ret;
2028         bool is_root = !getuid();
2029
2030         const auto log_status_on_exit = lttng::make_scope_exit([]() noexcept {
2031                 DBG_FMT("Kernel tracer status set to `{}`",
2032                         kernel_tracer_status_to_str(*kernel_tracer_status));
2033         });
2034
2035         /* Modprobe lttng kernel modules */
2036         ret = modprobe_lttng_control();
2037         if (ret < 0) {
2038                 set_kernel_tracer_status_from_modules_ret(-ret);
2039                 goto error;
2040         }
2041
2042         /* Open debugfs lttng */
2043         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
2044         if (kernel_tracer_fd < 0) {
2045                 DBG("Failed to open %s", module_proc_lttng);
2046                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2047                         LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG);
2048                 goto error_open;
2049         }
2050
2051         /* Validate kernel version */
2052         ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
2053         if (ret < 0) {
2054                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2055                         LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH);
2056                 goto error_version;
2057         }
2058
2059         ret = modprobe_lttng_data();
2060         if (ret < 0) {
2061                 set_kernel_tracer_status_from_modules_ret(-ret);
2062                 goto error_modules;
2063         }
2064
2065         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
2066         if (ret < 0) {
2067                 goto error_modules;
2068         }
2069         if (ret < 1) {
2070                 WARN("Kernel tracer does not support buffer monitoring. "
2071                      "The monitoring timer of channels in the kernel domain "
2072                      "will be set to 0 (disabled).");
2073         }
2074
2075         ret = kernel_supports_event_notifiers();
2076         if (ret < 0) {
2077                 ERR("Failed to check for kernel tracer event notifier support");
2078                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2079                         LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2080                 goto error_modules;
2081         }
2082         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2083         if (ret < 0) {
2084                 /* This is not fatal. */
2085                 WARN("Failed to create kernel event notifier group");
2086                 kernel_tracer_event_notifier_group_fd = -1;
2087         } else {
2088                 enum event_notifier_error_accounting_status error_accounting_status;
2089                 enum lttng_error_code error_code_ret =
2090                         kernel_create_event_notifier_group_notification_fd(
2091                                 &kernel_tracer_event_notifier_group_notification_fd);
2092
2093                 if (error_code_ret != LTTNG_OK) {
2094                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2095                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2096                         goto error_modules;
2097                 }
2098
2099                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2100                         kernel_tracer_event_notifier_group_fd);
2101                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2102                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2103                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2104                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2105                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2106                         goto error_modules;
2107                 }
2108
2109                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2110                         DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
2111                 if (!kernel_token_to_event_notifier_rule_ht) {
2112                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2113                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2114                         goto error_token_ht;
2115                 }
2116         }
2117
2118         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2119             kernel_tracer_fd,
2120             kernel_tracer_event_notifier_group_fd,
2121             kernel_tracer_event_notifier_group_notification_fd);
2122
2123         ret = syscall_init_table(kernel_tracer_fd);
2124         if (ret < 0) {
2125                 ERR("Unable to populate syscall table. Syscall tracing won't "
2126                     "work for this session daemon.");
2127         }
2128
2129         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2130                 LTTNG_KERNEL_TRACER_STATUS_INITIALIZED);
2131         return 0;
2132
2133 error_version:
2134         modprobe_remove_lttng_control();
2135         ret = close(kernel_tracer_fd);
2136         if (ret) {
2137                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2138         }
2139
2140         kernel_tracer_fd = -1;
2141         return LTTNG_ERR_KERN_VERSION;
2142
2143 error_token_ht:
2144         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2145         if (ret) {
2146                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2147                        kernel_tracer_event_notifier_group_notification_fd);
2148         }
2149
2150         kernel_tracer_event_notifier_group_notification_fd = -1;
2151
2152 error_modules:
2153         ret = close(kernel_tracer_event_notifier_group_fd);
2154         if (ret) {
2155                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2156                        kernel_tracer_event_notifier_group_fd);
2157         }
2158
2159         kernel_tracer_event_notifier_group_fd = -1;
2160
2161         ret = close(kernel_tracer_fd);
2162         if (ret) {
2163                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2164         }
2165
2166         kernel_tracer_fd = -1;
2167
2168 error_open:
2169         modprobe_remove_lttng_control();
2170
2171 error:
2172         WARN("No kernel tracer available");
2173         kernel_tracer_fd = -1;
2174         if (!is_root) {
2175                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2176                         LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT);
2177                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2178         } else {
2179                 return LTTNG_ERR_KERN_NA;
2180         }
2181 }
2182
2183 void cleanup_kernel_tracer()
2184 {
2185         DBG2("Closing kernel event notifier group notification file descriptor");
2186         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2187                 int ret = notification_thread_command_remove_tracer_event_source(
2188                         the_notification_thread_handle,
2189                         kernel_tracer_event_notifier_group_notification_fd);
2190                 if (ret != LTTNG_OK) {
2191                         ERR("Failed to remove kernel event notifier notification from notification thread");
2192                 }
2193
2194                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2195                 if (ret) {
2196                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2197                                kernel_tracer_event_notifier_group_notification_fd);
2198                 }
2199
2200                 kernel_tracer_event_notifier_group_notification_fd = -1;
2201         }
2202
2203         if (kernel_token_to_event_notifier_rule_ht) {
2204                 const int ret = cds_lfht_destroy(kernel_token_to_event_notifier_rule_ht, nullptr);
2205                 LTTNG_ASSERT(ret == 0);
2206         }
2207
2208         DBG2("Closing kernel event notifier group file descriptor");
2209         if (kernel_tracer_event_notifier_group_fd >= 0) {
2210                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2211
2212                 if (ret) {
2213                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2214                                kernel_tracer_event_notifier_group_fd);
2215                 }
2216
2217                 kernel_tracer_event_notifier_group_fd = -1;
2218         }
2219
2220         DBG2("Closing kernel fd");
2221         if (kernel_tracer_fd >= 0) {
2222                 const int ret = close(kernel_tracer_fd);
2223
2224                 if (ret) {
2225                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2226                                kernel_tracer_fd);
2227                 }
2228
2229                 kernel_tracer_fd = -1;
2230         }
2231
2232         kernel_tracer_status = nonstd::nullopt;
2233         free(syscall_table);
2234 }
2235
2236 bool kernel_tracer_is_initialized()
2237 {
2238         return kernel_tracer_fd >= 0;
2239 }
2240
2241 /*
2242  *  Clear a kernel session.
2243  *
2244  * Return LTTNG_OK on success or else an LTTng error code.
2245  */
2246 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2247 {
2248         int ret;
2249         enum lttng_error_code status = LTTNG_OK;
2250         struct consumer_socket *socket;
2251         struct lttng_ht_iter iter;
2252         struct ltt_kernel_session *ksess = session->kernel_session;
2253
2254         LTTNG_ASSERT(ksess);
2255         LTTNG_ASSERT(ksess->consumer);
2256
2257         DBG("Clear kernel session %s (session %" PRIu64 ")", session->name, session->id);
2258
2259         if (ksess->active) {
2260                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2261                 status = LTTNG_ERR_FATAL;
2262                 goto end;
2263         }
2264
2265         {
2266                 /*
2267                  * Note that this loop will end after one iteration given that there is
2268                  * only one kernel consumer.
2269                  */
2270                 lttng::urcu::read_lock_guard read_lock;
2271
2272                 cds_lfht_for_each_entry (
2273                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
2274                         struct ltt_kernel_channel *chan;
2275
2276                         /* For each channel, ask the consumer to clear it. */
2277                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
2278                                 DBG("Clear kernel channel %" PRIu64 ", session %s",
2279                                     chan->key,
2280                                     session->name);
2281                                 ret = consumer_clear_channel(socket, chan->key);
2282                                 if (ret < 0) {
2283                                         goto error;
2284                                 }
2285                         }
2286
2287                         if (!ksess->metadata) {
2288                                 /*
2289                                  * Nothing to do for the metadata.
2290                                  * This is a snapshot session.
2291                                  * The metadata is genererated on the fly.
2292                                  */
2293                                 continue;
2294                         }
2295
2296                         /*
2297                          * Clear the metadata channel.
2298                          * Metadata channel is not cleared per se but we still need to
2299                          * perform a rotation operation on it behind the scene.
2300                          */
2301                         ret = consumer_clear_channel(socket, ksess->metadata->key);
2302                         if (ret < 0) {
2303                                 goto error;
2304                         }
2305                 }
2306         }
2307
2308         goto end;
2309 error:
2310         switch (-ret) {
2311         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2312                 status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2313                 break;
2314         default:
2315                 status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2316                 break;
2317         }
2318 end:
2319         return status;
2320 }
2321
2322 enum lttng_error_code
2323 kernel_create_event_notifier_group_notification_fd(int *event_notifier_group_notification_fd)
2324 {
2325         int local_fd = -1, ret;
2326         enum lttng_error_code error_code_ret;
2327
2328         LTTNG_ASSERT(event_notifier_group_notification_fd);
2329
2330         ret = kernctl_create_event_notifier_group_notification_fd(
2331                 kernel_tracer_event_notifier_group_fd);
2332         if (ret < 0) {
2333                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2334                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2335                 goto error;
2336         }
2337
2338         local_fd = ret;
2339
2340         /* Prevent fd duplication after execlp(). */
2341         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2342         if (ret < 0) {
2343                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2344                        local_fd);
2345                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2346                 goto error;
2347         }
2348
2349         DBG("Created kernel notifier group notification file descriptor: fd = %d", local_fd);
2350         error_code_ret = LTTNG_OK;
2351         *event_notifier_group_notification_fd = local_fd;
2352         local_fd = -1;
2353
2354 error:
2355         if (local_fd >= 0) {
2356                 ret = close(local_fd);
2357                 if (ret) {
2358                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2359                                local_fd);
2360                 }
2361         }
2362
2363         return error_code_ret;
2364 }
2365
2366 enum lttng_error_code
2367 kernel_destroy_event_notifier_group_notification_fd(int event_notifier_group_notification_fd)
2368 {
2369         enum lttng_error_code ret_code = LTTNG_OK;
2370
2371         DBG("Closing event notifier group notification file descriptor: fd = %d",
2372             event_notifier_group_notification_fd);
2373         if (event_notifier_group_notification_fd >= 0) {
2374                 const int ret = close(event_notifier_group_notification_fd);
2375                 if (ret) {
2376                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2377                                event_notifier_group_notification_fd);
2378                 }
2379         }
2380
2381         return ret_code;
2382 }
2383
2384 static unsigned long hash_trigger(const struct lttng_trigger *trigger)
2385 {
2386         const struct lttng_condition *condition = lttng_trigger_get_const_condition(trigger);
2387
2388         return lttng_condition_hash(condition);
2389 }
2390
2391 static int match_trigger(struct cds_lfht_node *node, const void *key)
2392 {
2393         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2394         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2395
2396         event_notifier_rule =
2397                 caa_container_of(node, const struct ltt_kernel_event_notifier_rule, ht_node);
2398
2399         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2400 }
2401
2402 static enum lttng_error_code kernel_create_event_notifier_rule(
2403         struct lttng_trigger *trigger, const struct lttng_credentials *creds, uint64_t token)
2404 {
2405         int err, fd, ret = 0;
2406         enum lttng_error_code error_code_ret;
2407         enum lttng_condition_status condition_status;
2408         enum lttng_condition_type condition_type;
2409         enum lttng_event_rule_type event_rule_type;
2410         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2411         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2412         unsigned int capture_bytecode_count = 0, i;
2413         const struct lttng_condition *condition = nullptr;
2414         const struct lttng_event_rule *event_rule = nullptr;
2415         enum lttng_condition_status cond_status;
2416
2417         LTTNG_ASSERT(trigger);
2418
2419         condition = lttng_trigger_get_const_condition(trigger);
2420         LTTNG_ASSERT(condition);
2421
2422         condition_type = lttng_condition_get_type(condition);
2423         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2424
2425         /* Does not acquire a reference. */
2426         condition_status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2427         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2428         LTTNG_ASSERT(event_rule);
2429
2430         event_rule_type = lttng_event_rule_get_type(event_rule);
2431         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2432
2433         error_code_ret = trace_kernel_create_event_notifier_rule(
2434                 trigger,
2435                 token,
2436                 lttng_condition_event_rule_matches_get_error_counter_index(condition),
2437                 &event_notifier_rule);
2438         if (error_code_ret != LTTNG_OK) {
2439                 goto error;
2440         }
2441
2442         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(event_rule,
2443                                                                           &kernel_event_notifier);
2444         if (error_code_ret != LTTNG_OK) {
2445                 goto free_event;
2446         }
2447
2448         kernel_event_notifier.event.token = event_notifier_rule->token;
2449         kernel_event_notifier.error_counter_idx =
2450                 lttng_condition_event_rule_matches_get_error_counter_index(condition);
2451
2452         fd = kernctl_create_event_notifier(kernel_tracer_event_notifier_group_fd,
2453                                            &kernel_event_notifier);
2454         if (fd < 0) {
2455                 switch (-fd) {
2456                 case EEXIST:
2457                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2458                         break;
2459                 case ENOSYS:
2460                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2461                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2462                         break;
2463                 case ENOENT:
2464                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2465                              kernel_event_notifier.event.name);
2466                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2467                         break;
2468                 default:
2469                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2470                                fd,
2471                                kernel_event_notifier.event.name);
2472                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2473                 }
2474                 goto free_event;
2475         }
2476
2477         event_notifier_rule->fd = fd;
2478         /* Prevent fd duplication after execlp(). */
2479         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2480         if (err < 0) {
2481                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2482                        fd);
2483                 error_code_ret = LTTNG_ERR_FATAL;
2484                 goto set_cloexec_error;
2485         }
2486
2487         if (event_notifier_rule->filter) {
2488                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2489                 if (err < 0) {
2490                         switch (-err) {
2491                         case ENOMEM:
2492                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2493                                 break;
2494                         default:
2495                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2496                                 break;
2497                         }
2498                         goto filter_error;
2499                 }
2500         }
2501
2502         if (lttng_event_rule_get_type(event_rule) == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2503                 ret = userspace_probe_event_rule_add_callsites(
2504                         event_rule, creds, event_notifier_rule->fd);
2505                 if (ret) {
2506                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2507                         goto add_callsite_error;
2508                 }
2509         }
2510
2511         /* Set the capture bytecode if any. */
2512         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2513                 condition, &capture_bytecode_count);
2514         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2515
2516         for (i = 0; i < capture_bytecode_count; i++) {
2517                 const struct lttng_bytecode *capture_bytecode =
2518                         lttng_condition_event_rule_matches_get_capture_bytecode_at_index(condition,
2519                                                                                          i);
2520
2521                 if (capture_bytecode == nullptr) {
2522                         ERR("Unexpected NULL capture bytecode on condition");
2523                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2524                         goto capture_error;
2525                 }
2526
2527                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2528                 if (ret < 0) {
2529                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2530                             event_notifier_rule->fd);
2531                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2532                         goto capture_error;
2533                 }
2534         }
2535
2536         err = kernctl_enable(event_notifier_rule->fd);
2537         if (err < 0) {
2538                 switch (-err) {
2539                 case EEXIST:
2540                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2541                         break;
2542                 default:
2543                         PERROR("enable kernel event notifier");
2544                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2545                         break;
2546                 }
2547                 goto enable_error;
2548         }
2549
2550         /* Add trigger to kernel token mapping in the hash table. */
2551         {
2552                 lttng::urcu::read_lock_guard read_lock;
2553                 cds_lfht_add(kernel_token_to_event_notifier_rule_ht,
2554                              hash_trigger(trigger),
2555                              &event_notifier_rule->ht_node);
2556         }
2557
2558         DBG("Created kernel event notifier: name = '%s', fd = %d",
2559             kernel_event_notifier.event.name,
2560             event_notifier_rule->fd);
2561
2562         return LTTNG_OK;
2563
2564 capture_error:
2565 add_callsite_error:
2566 enable_error:
2567 set_cloexec_error:
2568 filter_error:
2569 {
2570         const int close_ret = close(event_notifier_rule->fd);
2571
2572         if (close_ret) {
2573                 PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2574                        event_notifier_rule->fd);
2575         }
2576 }
2577 free_event:
2578         free(event_notifier_rule);
2579 error:
2580         return error_code_ret;
2581 }
2582
2583 enum lttng_error_code kernel_register_event_notifier(struct lttng_trigger *trigger,
2584                                                      const struct lttng_credentials *cmd_creds)
2585 {
2586         enum lttng_error_code ret;
2587         enum lttng_condition_status status;
2588         enum lttng_domain_type domain_type;
2589         const struct lttng_event_rule *event_rule;
2590         const struct lttng_condition *const condition = lttng_trigger_get_const_condition(trigger);
2591         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2592
2593         LTTNG_ASSERT(condition);
2594
2595         /* Does not acquire a reference to the event rule. */
2596         status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2597         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2598
2599         domain_type = lttng_event_rule_get_domain_type(event_rule);
2600         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2601
2602         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2603         if (ret != LTTNG_OK) {
2604                 ERR("Failed to create kernel event notifier rule");
2605         }
2606
2607         return ret;
2608 }
2609
2610 enum lttng_error_code kernel_unregister_event_notifier(const struct lttng_trigger *trigger)
2611 {
2612         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2613         struct cds_lfht_node *node;
2614         struct cds_lfht_iter iter;
2615         enum lttng_error_code error_code_ret;
2616         int ret;
2617
2618         lttng::urcu::read_lock_guard read_lock;
2619
2620         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2621                         hash_trigger(trigger),
2622                         match_trigger,
2623                         trigger,
2624                         &iter);
2625
2626         node = cds_lfht_iter_get_node(&iter);
2627         if (!node) {
2628                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2629                 goto error;
2630         }
2631
2632         token_event_rule_element =
2633                 caa_container_of(node, struct ltt_kernel_event_notifier_rule, ht_node);
2634
2635         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2636         if (ret) {
2637                 error_code_ret = LTTNG_ERR_FATAL;
2638                 goto error;
2639         }
2640
2641         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2642         error_code_ret = LTTNG_OK;
2643
2644 error:
2645
2646         return error_code_ret;
2647 }
2648
2649 int kernel_get_notification_fd()
2650 {
2651         return kernel_tracer_event_notifier_group_notification_fd;
2652 }