src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include "condition-internal.hpp"
  10 #include "consumer.hpp"
  11 #include "event-notifier-error-accounting.hpp"
  12 #include "kern-modules.hpp"
  13 #include "kernel-consumer.hpp"
  14 #include "kernel.hpp"
  15 #include "lttng-sessiond.hpp"
  16 #include "lttng-syscall.hpp"
  17 #include "modprobe.hpp"
  18 #include "notification-thread-commands.hpp"
  19 #include "rotate.hpp"
  20 #include "sessiond-config.hpp"
  21 #include "tracker.hpp"
  22 #include "utils.hpp"
  23
  24 #include <common/common.hpp>
  25 #include <common/hashtable/utils.hpp>
  26 #include <common/kernel-ctl/kernel-ctl.hpp>
  27 #include <common/kernel-ctl/kernel-ioctl.hpp>
  28 #include <common/sessiond-comm/sessiond-comm.hpp>
  29 #include <common/trace-chunk.hpp>
  30 #include <common/tracker.hpp>
  31 #include <common/urcu.hpp>
  32 #include <common/utils.hpp>
  33
  34 #include <lttng/condition/event-rule-matches-internal.hpp>
  35 #include <lttng/condition/event-rule-matches.h>
  36 #include <lttng/event-rule/event-rule-internal.hpp>
  37 #include <lttng/event-rule/event-rule.h>
  38 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  39 #include <lttng/event.h>
  40 #include <lttng/lttng-error.h>
  41 #include <lttng/tracker.h>
  42 #include <lttng/userspace-probe-internal.hpp>
  43 #include <lttng/userspace-probe.h>
  44
  45 #include <fcntl.h>
  46 #include <inttypes.h>
  47 #include <stdio.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <sys/types.h>
  51 #include <unistd.h>
  52
  53 /*
  54  * Key used to reference a channel between the sessiond and the consumer. This
  55  * is only read and updated with the session_list lock held.
  56  */
  57 static uint64_t next_kernel_channel_key;
  58
  59 static const char *module_proc_lttng = "/proc/lttng";
  60
  61 static int kernel_tracer_fd = -1;
  62 static int kernel_tracer_event_notifier_group_fd = -1;
  63 static int kernel_tracer_event_notifier_group_notification_fd = -1;
  64 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  65
  66 namespace {
  67 /*
  68  * On some architectures, calling convention details are embedded in the symbol
  69  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  70  * where an instruction boundary would be legal) to add
  71  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  72  * a per-architecture basis.
  73  */
  74 #if defined(__arm__) || defined(__aarch64__)
  75 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  76 {
  77         /*
  78          * The least significant bit is used when branching to switch to thumb
  79          * ISA. However, it's an invalid address for us; mask the least
  80          * significant bit.
  81          */
  82         return raw_offset &= ~0b1;
  83 }
  84 #else /* defined(__arm__) || defined(__aarch64__) */
  85 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  86 {
  87         return raw_offset;
  88 }
  89 #endif
  90 }
  91
  92 /*
  93  * Add context on a kernel channel.
  94  *
  95  * Assumes the ownership of ctx.
  96  */
  97 int kernel_add_channel_context(struct ltt_kernel_channel *chan, struct ltt_kernel_context *ctx)
  98 {
  99         int ret;
 100
 101         LTTNG_ASSERT(chan);
 102         LTTNG_ASSERT(ctx);
 103
 104         DBG("Adding context to channel %s", chan->channel->name);
 105         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 106         if (ret < 0) {
 107                 switch (-ret) {
 108                 case ENOSYS:
 109                         /* Exists but not available for this kernel */
 110                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 111                         goto error;
 112                 case EEXIST:
 113                         /* If EEXIST, we just ignore the error */
 114                         ret = 0;
 115                         goto end;
 116                 default:
 117                         PERROR("add context ioctl");
 118                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 119                         goto error;
 120                 }
 121         }
 122         ret = 0;
 123
 124 end:
 125         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 126         ctx->in_list = true;
 127         ctx = nullptr;
 128 error:
 129         if (ctx) {
 130                 trace_kernel_destroy_context(ctx);
 131         }
 132         return ret;
 133 }
 134
 135 /*
 136  * Create a new kernel session, register it to the kernel tracer and add it to
 137  * the session daemon session.
 138  */
 139 int kernel_create_session(struct ltt_session *session)
 140 {
 141         int ret;
 142         struct ltt_kernel_session *lks;
 143
 144         LTTNG_ASSERT(session);
 145
 146         /* Allocate data structure */
 147         lks = trace_kernel_create_session();
 148         if (lks == nullptr) {
 149                 ret = -1;
 150                 goto error;
 151         }
 152
 153         /* Kernel tracer session creation */
 154         ret = kernctl_create_session(kernel_tracer_fd);
 155         if (ret < 0) {
 156                 PERROR("ioctl kernel create session");
 157                 goto error;
 158         }
 159
 160         lks->fd = ret;
 161         /* Prevent fd duplication after execlp() */
 162         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 163         if (ret < 0) {
 164                 PERROR("fcntl session fd");
 165         }
 166
 167         lks->id = session->id;
 168         lks->consumer_fds_sent = 0;
 169         session->kernel_session = lks;
 170
 171         DBG("Kernel session created (fd: %d)", lks->fd);
 172
 173         /*
 174          * This is necessary since the creation time is present in the session
 175          * name when it is generated.
 176          */
 177         if (session->has_auto_generated_name) {
 178                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 179         } else {
 180                 ret = kernctl_session_set_name(lks->fd, session->name);
 181         }
 182         if (ret) {
 183                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 184                      session->id,
 185                      session->name);
 186         }
 187
 188         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 189         if (ret) {
 190                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 191                      session->id,
 192                      session->name);
 193         }
 194
 195         return 0;
 196
 197 error:
 198         if (lks) {
 199                 trace_kernel_destroy_session(lks);
 200                 trace_kernel_free_session(lks);
 201         }
 202         return ret;
 203 }
 204
 205 /*
 206  * Create a kernel channel, register it to the kernel tracer and add it to the
 207  * kernel session.
 208  */
 209 int kernel_create_channel(struct ltt_kernel_session *session, struct lttng_channel *chan)
 210 {
 211         int ret;
 212         struct ltt_kernel_channel *lkc;
 213
 214         LTTNG_ASSERT(session);
 215         LTTNG_ASSERT(chan);
 216
 217         /* Allocate kernel channel */
 218         lkc = trace_kernel_create_channel(chan);
 219         if (lkc == nullptr) {
 220                 goto error;
 221         }
 222
 223         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 224              chan->name,
 225              lkc->channel->attr.overwrite,
 226              lkc->channel->attr.subbuf_size,
 227              lkc->channel->attr.num_subbuf,
 228              lkc->channel->attr.switch_timer_interval,
 229              lkc->channel->attr.read_timer_interval,
 230              lkc->channel->attr.live_timer_interval,
 231              lkc->channel->attr.output);
 232
 233         /* Kernel tracer channel creation */
 234         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 235         if (ret < 0) {
 236                 PERROR("ioctl kernel create channel");
 237                 goto error;
 238         }
 239
 240         /* Setup the channel fd */
 241         lkc->fd = ret;
 242         /* Prevent fd duplication after execlp() */
 243         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 244         if (ret < 0) {
 245                 PERROR("fcntl session fd");
 246         }
 247
 248         /* Add channel to session */
 249         cds_list_add(&lkc->list, &session->channel_list.head);
 250         session->channel_count++;
 251         lkc->session = session;
 252         lkc->key = ++next_kernel_channel_key;
 253
 254         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 255             lkc->channel->name,
 256             lkc->fd,
 257             lkc->key);
 258
 259         return 0;
 260
 261 error:
 262         if (lkc) {
 263                 free(lkc->channel);
 264                 free(lkc);
 265         }
 266         return -1;
 267 }
 268
 269 /*
 270  * Create a kernel event notifier group, register it to the kernel tracer and
 271  * add it to the kernel session.
 272  */
 273 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 274 {
 275         int ret;
 276         int local_fd = -1;
 277
 278         LTTNG_ASSERT(event_notifier_group_fd);
 279
 280         /* Kernel event notifier group creation. */
 281         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 282         if (ret < 0) {
 283                 PERROR("Failed to create kernel event notifier group");
 284                 ret = -1;
 285                 goto error;
 286         }
 287
 288         local_fd = ret;
 289
 290         /* Prevent fd duplication after execlp(). */
 291         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 292         if (ret < 0) {
 293                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 294                        local_fd);
 295                 goto error;
 296         }
 297
 298         DBG("Created kernel event notifier group: fd = %d", local_fd);
 299         *event_notifier_group_fd = local_fd;
 300         local_fd = -1;
 301         ret = 0;
 302 error:
 303         if (local_fd >= 0) {
 304                 ret = close(local_fd);
 305                 if (ret) {
 306                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 307                                local_fd);
 308                 }
 309         }
 310
 311         return ret;
 312 }
 313
 314 /*
 315  * Compute the offset of the instrumentation byte in the binary based on the
 316  * function probe location using the ELF lookup method.
 317  *
 318  * Returns 0 on success and set the offset out parameter to the offset of the
 319  * elf symbol
 320  * Returns -1 on error
 321  */
 322 static int extract_userspace_probe_offset_function_elf(
 323         const struct lttng_userspace_probe_location *probe_location,
 324         uid_t uid,
 325         gid_t gid,
 326         uint64_t *offset)
 327 {
 328         int fd;
 329         int ret = 0;
 330         const char *symbol = nullptr;
 331         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 332         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 333
 334         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 335                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 336
 337         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 338         if (!lookup) {
 339                 ret = -1;
 340                 goto end;
 341         }
 342
 343         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 344
 345         LTTNG_ASSERT(lookup_method_type ==
 346                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 347
 348         symbol = lttng_userspace_probe_location_function_get_function_name(probe_location);
 349         if (!symbol) {
 350                 ret = -1;
 351                 goto end;
 352         }
 353
 354         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 355         if (fd < 0) {
 356                 ret = -1;
 357                 goto end;
 358         }
 359
 360         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 361         if (ret < 0) {
 362                 DBG("userspace probe offset calculation failed for "
 363                     "function %s",
 364                     symbol);
 365                 goto end;
 366         }
 367
 368         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t) (*offset));
 369 end:
 370         return ret;
 371 }
 372
 373 /*
 374  * Compute the offsets of the instrumentation bytes in the binary based on the
 375  * tracepoint probe location using the SDT lookup method. This function
 376  * allocates the offsets buffer, the caller must free it.
 377  *
 378  * Returns 0 on success and set the offset out parameter to the offsets of the
 379  * SDT tracepoint.
 380  * Returns -1 on error.
 381  */
 382 static int extract_userspace_probe_offset_tracepoint_sdt(
 383         const struct lttng_userspace_probe_location *probe_location,
 384         uid_t uid,
 385         gid_t gid,
 386         uint64_t **offsets,
 387         uint32_t *offsets_count)
 388 {
 389         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 390         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 391         const char *probe_name = nullptr, *provider_name = nullptr;
 392         int ret = 0;
 393         int fd, i;
 394
 395         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 396                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 397
 398         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 399         if (!lookup) {
 400                 ret = -1;
 401                 goto end;
 402         }
 403
 404         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 405
 406         LTTNG_ASSERT(lookup_method_type ==
 407                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 408
 409         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(probe_location);
 410         if (!probe_name) {
 411                 ret = -1;
 412                 goto end;
 413         }
 414
 415         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(probe_location);
 416         if (!provider_name) {
 417                 ret = -1;
 418                 goto end;
 419         }
 420
 421         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 422         if (fd < 0) {
 423                 ret = -1;
 424                 goto end;
 425         }
 426
 427         ret = run_as_extract_sdt_probe_offsets(
 428                 fd, provider_name, probe_name, uid, gid, offsets, offsets_count);
 429         if (ret < 0) {
 430                 DBG("userspace probe offset calculation failed for sdt "
 431                     "probe %s:%s",
 432                     provider_name,
 433                     probe_name);
 434                 goto end;
 435         }
 436
 437         if (*offsets_count == 0) {
 438                 DBG("no userspace probe offset found");
 439                 goto end;
 440         }
 441
 442         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 443             *offsets_count,
 444             provider_name,
 445             probe_name);
 446         for (i = 0; i < *offsets_count; i++) {
 447                 DBG("\t0x%jd", (intmax_t) ((*offsets)[i]));
 448         }
 449 end:
 450         return ret;
 451 }
 452
 453 static int userspace_probe_add_callsite(const struct lttng_userspace_probe_location *location,
 454                                         uid_t uid,
 455                                         gid_t gid,
 456                                         int fd)
 457 {
 458         const struct lttng_userspace_probe_location_lookup_method *lookup_method = nullptr;
 459         enum lttng_userspace_probe_location_lookup_method_type type;
 460         int ret;
 461
 462         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 463         if (!lookup_method) {
 464                 ret = -1;
 465                 goto end;
 466         }
 467
 468         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 469         switch (type) {
 470         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 471         {
 472                 struct lttng_kernel_abi_event_callsite callsite;
 473                 uint64_t offset;
 474
 475                 ret = extract_userspace_probe_offset_function_elf(location, uid, gid, &offset);
 476                 if (ret) {
 477                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 478                         goto end;
 479                 }
 480
 481                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 482                 ret = kernctl_add_callsite(fd, &callsite);
 483                 if (ret) {
 484                         WARN("Failed to add callsite to ELF userspace probe.");
 485                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 486                         goto end;
 487                 }
 488                 break;
 489         }
 490         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 491         {
 492                 int i;
 493                 uint64_t *offsets = nullptr;
 494                 uint32_t offsets_count;
 495                 struct lttng_kernel_abi_event_callsite callsite;
 496
 497                 /*
 498                  * This call allocates the offsets buffer. This buffer must be freed
 499                  * by the caller
 500                  */
 501                 ret = extract_userspace_probe_offset_tracepoint_sdt(
 502                         location, uid, gid, &offsets, &offsets_count);
 503                 if (ret) {
 504                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 505                         goto end;
 506                 }
 507                 for (i = 0; i < offsets_count; i++) {
 508                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 509                         ret = kernctl_add_callsite(fd, &callsite);
 510                         if (ret) {
 511                                 WARN("Failed to add callsite to SDT userspace probe");
 512                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 513                                 free(offsets);
 514                                 goto end;
 515                         }
 516                 }
 517                 free(offsets);
 518                 break;
 519         }
 520         default:
 521                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 522                 goto end;
 523         }
 524 end:
 525         return ret;
 526 }
 527
 528 /*
 529  * Extract the offsets of the instrumentation point for the different lookup
 530  * methods.
 531  */
 532 static int userspace_probe_event_add_callsites(struct lttng_event *ev,
 533                                                struct ltt_kernel_session *session,
 534                                                int fd)
 535 {
 536         int ret;
 537         const struct lttng_userspace_probe_location *location = nullptr;
 538
 539         LTTNG_ASSERT(ev);
 540         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 541
 542         location = lttng_event_get_userspace_probe_location(ev);
 543         if (!location) {
 544                 ret = -1;
 545                 goto end;
 546         }
 547
 548         ret = userspace_probe_add_callsite(location, session->uid, session->gid, fd);
 549         if (ret) {
 550                 WARN("Failed to add callsite to userspace probe event '%s'", ev->name);
 551         }
 552
 553 end:
 554         return ret;
 555 }
 556
 557 /*
 558  * Extract the offsets of the instrumentation point for the different look-up
 559  * methods.
 560  */
 561 static int userspace_probe_event_rule_add_callsites(const struct lttng_event_rule *rule,
 562                                                     const struct lttng_credentials *creds,
 563                                                     int fd)
 564 {
 565         int ret;
 566         enum lttng_event_rule_status status;
 567         enum lttng_event_rule_type event_rule_type;
 568         const struct lttng_userspace_probe_location *location = nullptr;
 569
 570         LTTNG_ASSERT(rule);
 571         LTTNG_ASSERT(creds);
 572
 573         event_rule_type = lttng_event_rule_get_type(rule);
 574         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 575
 576         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 577         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 578                 ret = -1;
 579                 goto end;
 580         }
 581
 582         ret = userspace_probe_add_callsite(
 583                 location, lttng_credentials_get_uid(creds), lttng_credentials_get_gid(creds), fd);
 584         if (ret) {
 585                 WARN("Failed to add callsite to user space probe object: fd = %d", fd);
 586         }
 587
 588 end:
 589         return ret;
 590 }
 591
 592 /*
 593  * Create a kernel event, enable it to the kernel tracer and add it to the
 594  * channel event list of the kernel session.
 595  * We own filter_expression and filter.
 596  */
 597 int kernel_create_event(struct lttng_event *ev,
 598                         struct ltt_kernel_channel *channel,
 599                         char *filter_expression,
 600                         struct lttng_bytecode *filter)
 601 {
 602         int err, fd;
 603         enum lttng_error_code ret;
 604         struct ltt_kernel_event *event;
 605
 606         LTTNG_ASSERT(ev);
 607         LTTNG_ASSERT(channel);
 608
 609         /* We pass ownership of filter_expression and filter */
 610         ret = trace_kernel_create_event(ev, filter_expression, filter, &event);
 611         if (ret != LTTNG_OK) {
 612                 goto error;
 613         }
 614
 615         fd = kernctl_create_event(channel->fd, event->event);
 616         if (fd < 0) {
 617                 switch (-fd) {
 618                 case EEXIST:
 619                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 620                         break;
 621                 case ENOSYS:
 622                         WARN("Event type not implemented");
 623                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 624                         break;
 625                 case ENOENT:
 626                         WARN("Event %s not found!", ev->name);
 627                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 628                         break;
 629                 default:
 630                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 631                         PERROR("create event ioctl");
 632                 }
 633                 goto free_event;
 634         }
 635
 636         event->type = ev->type;
 637         event->fd = fd;
 638         /* Prevent fd duplication after execlp() */
 639         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 640         if (err < 0) {
 641                 PERROR("fcntl session fd");
 642         }
 643
 644         if (filter) {
 645                 err = kernctl_filter(event->fd, filter);
 646                 if (err < 0) {
 647                         switch (-err) {
 648                         case ENOMEM:
 649                                 ret = LTTNG_ERR_FILTER_NOMEM;
 650                                 break;
 651                         default:
 652                                 ret = LTTNG_ERR_FILTER_INVAL;
 653                                 break;
 654                         }
 655                         goto filter_error;
 656                 }
 657         }
 658
 659         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 660                 ret = (lttng_error_code) userspace_probe_event_add_callsites(
 661                         ev, channel->session, event->fd);
 662                 if (ret) {
 663                         goto add_callsite_error;
 664                 }
 665         }
 666
 667         err = kernctl_enable(event->fd);
 668         if (err < 0) {
 669                 switch (-err) {
 670                 case EEXIST:
 671                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 672                         break;
 673                 default:
 674                         PERROR("enable kernel event");
 675                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 676                         break;
 677                 }
 678                 goto enable_error;
 679         }
 680
 681         /* Add event to event list */
 682         cds_list_add(&event->list, &channel->events_list.head);
 683         channel->event_count++;
 684
 685         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 686
 687         return 0;
 688
 689 add_callsite_error:
 690 enable_error:
 691 filter_error:
 692 {
 693         int closeret;
 694
 695         closeret = close(event->fd);
 696         if (closeret) {
 697                 PERROR("close event fd");
 698         }
 699 }
 700 free_event:
 701         free(event);
 702 error:
 703         return ret;
 704 }
 705
 706 /*
 707  * Disable a kernel channel.
 708  */
 709 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 710 {
 711         int ret;
 712
 713         LTTNG_ASSERT(chan);
 714
 715         ret = kernctl_disable(chan->fd);
 716         if (ret < 0) {
 717                 PERROR("disable chan ioctl");
 718                 goto error;
 719         }
 720
 721         chan->enabled = false;
 722         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 723             chan->channel->name,
 724             chan->fd,
 725             chan->key);
 726
 727         return 0;
 728
 729 error:
 730         return ret;
 731 }
 732
 733 /*
 734  * Enable a kernel channel.
 735  */
 736 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 737 {
 738         int ret;
 739
 740         LTTNG_ASSERT(chan);
 741
 742         ret = kernctl_enable(chan->fd);
 743         if (ret < 0 && ret != -EEXIST) {
 744                 PERROR("Enable kernel chan");
 745                 goto error;
 746         }
 747
 748         chan->enabled = true;
 749         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 750             chan->channel->name,
 751             chan->fd,
 752             chan->key);
 753
 754         return 0;
 755
 756 error:
 757         return ret;
 758 }
 759
 760 /*
 761  * Enable a kernel event.
 762  */
 763 int kernel_enable_event(struct ltt_kernel_event *event)
 764 {
 765         int ret;
 766
 767         LTTNG_ASSERT(event);
 768
 769         ret = kernctl_enable(event->fd);
 770         if (ret < 0) {
 771                 switch (-ret) {
 772                 case EEXIST:
 773                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 774                         break;
 775                 default:
 776                         PERROR("enable kernel event");
 777                         break;
 778                 }
 779                 goto error;
 780         }
 781
 782         event->enabled = true;
 783         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 784
 785         return 0;
 786
 787 error:
 788         return ret;
 789 }
 790
 791 /*
 792  * Disable a kernel event.
 793  */
 794 int kernel_disable_event(struct ltt_kernel_event *event)
 795 {
 796         int ret;
 797
 798         LTTNG_ASSERT(event);
 799
 800         ret = kernctl_disable(event->fd);
 801         if (ret < 0) {
 802                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 803                        event->event->name,
 804                        event->fd);
 805                 goto error;
 806         }
 807
 808         event->enabled = false;
 809         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 810
 811         return 0;
 812
 813 error:
 814         return ret;
 815 }
 816
 817 /*
 818  * Disable a kernel event notifier.
 819  */
 820 static int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 821 {
 822         int ret;
 823
 824         LTTNG_ASSERT(event);
 825
 826         lttng::urcu::read_lock_guard read_lock;
 827         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 828
 829         ret = kernctl_disable(event->fd);
 830         if (ret < 0) {
 831                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 832                        event->fd,
 833                        event->token);
 834                 goto error;
 835         }
 836
 837         event->enabled = false;
 838         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64, event->fd, event->token);
 839
 840 error:
 841         return ret;
 842 }
 843
 844 static struct process_attr_tracker *
 845 _kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 846                                  enum lttng_process_attr process_attr)
 847 {
 848         switch (process_attr) {
 849         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 850                 return session->tracker_pid;
 851         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 852                 return session->tracker_vpid;
 853         case LTTNG_PROCESS_ATTR_USER_ID:
 854                 return session->tracker_uid;
 855         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 856                 return session->tracker_vuid;
 857         case LTTNG_PROCESS_ATTR_GROUP_ID:
 858                 return session->tracker_gid;
 859         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 860                 return session->tracker_vgid;
 861         default:
 862                 return nullptr;
 863         }
 864 }
 865
 866 const struct process_attr_tracker *
 867 kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 868                                 enum lttng_process_attr process_attr)
 869 {
 870         return (const struct process_attr_tracker *) _kernel_get_process_attr_tracker(session,
 871                                                                                       process_attr);
 872 }
 873
 874 enum lttng_error_code
 875 kernel_process_attr_tracker_set_tracking_policy(struct ltt_kernel_session *session,
 876                                                 enum lttng_process_attr process_attr,
 877                                                 enum lttng_tracking_policy policy)
 878 {
 879         int ret;
 880         enum lttng_error_code ret_code = LTTNG_OK;
 881         struct process_attr_tracker *tracker =
 882                 _kernel_get_process_attr_tracker(session, process_attr);
 883         enum lttng_tracking_policy previous_policy;
 884
 885         if (!tracker) {
 886                 ret_code = LTTNG_ERR_INVALID;
 887                 goto end;
 888         }
 889
 890         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 891         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 892         if (ret) {
 893                 ret_code = LTTNG_ERR_UNK;
 894                 goto end;
 895         }
 896
 897         if (previous_policy == policy) {
 898                 goto end;
 899         }
 900
 901         switch (policy) {
 902         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 903                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 904                         /*
 905                          * Maintain a special case for the process ID process
 906                          * attribute tracker as it was the only supported
 907                          * attribute prior to 2.12.
 908                          */
 909                         ret = kernctl_track_pid(session->fd, -1);
 910                 } else {
 911                         ret = kernctl_track_id(session->fd, process_attr, -1);
 912                 }
 913                 break;
 914         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 915         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 916                 /* fall-through. */
 917                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 918                         /*
 919                          * Maintain a special case for the process ID process
 920                          * attribute tracker as it was the only supported
 921                          * attribute prior to 2.12.
 922                          */
 923                         ret = kernctl_untrack_pid(session->fd, -1);
 924                 } else {
 925                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 926                 }
 927                 break;
 928         default:
 929                 abort();
 930         }
 931         /* kern-ctl error handling */
 932         switch (-ret) {
 933         case 0:
 934                 ret_code = LTTNG_OK;
 935                 break;
 936         case EINVAL:
 937                 ret_code = LTTNG_ERR_INVALID;
 938                 break;
 939         case ENOMEM:
 940                 ret_code = LTTNG_ERR_NOMEM;
 941                 break;
 942         case EEXIST:
 943                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 944                 break;
 945         default:
 946                 ret_code = LTTNG_ERR_UNK;
 947                 break;
 948         }
 949 end:
 950         return ret_code;
 951 }
 952
 953 enum lttng_error_code
 954 kernel_process_attr_tracker_inclusion_set_add_value(struct ltt_kernel_session *session,
 955                                                     enum lttng_process_attr process_attr,
 956                                                     const struct process_attr_value *value)
 957 {
 958         int ret, integral_value;
 959         enum lttng_error_code ret_code;
 960         struct process_attr_tracker *tracker;
 961         enum process_attr_tracker_status status;
 962
 963         /*
 964          * Convert process attribute tracker value to the integral
 965          * representation required by the kern-ctl API.
 966          */
 967         switch (process_attr) {
 968         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 969         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 970                 integral_value = (int) value->value.pid;
 971                 break;
 972         case LTTNG_PROCESS_ATTR_USER_ID:
 973         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 974                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 975                         uid_t uid;
 976
 977                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
 978                         if (ret_code != LTTNG_OK) {
 979                                 goto end;
 980                         }
 981                         integral_value = (int) uid;
 982                 } else {
 983                         integral_value = (int) value->value.uid;
 984                 }
 985                 break;
 986         case LTTNG_PROCESS_ATTR_GROUP_ID:
 987         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 988                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 989                         gid_t gid;
 990
 991                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
 992                         if (ret_code != LTTNG_OK) {
 993                                 goto end;
 994                         }
 995                         integral_value = (int) gid;
 996                 } else {
 997                         integral_value = (int) value->value.gid;
 998                 }
 999                 break;
1000         default:
1001                 ret_code = LTTNG_ERR_INVALID;
1002                 goto end;
1003         }
1004
1005         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1006         if (!tracker) {
1007                 ret_code = LTTNG_ERR_INVALID;
1008                 goto end;
1009         }
1010
1011         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1012         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1013                 switch (status) {
1014                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1015                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1016                         break;
1017                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1018                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1019                         break;
1020                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1021                 default:
1022                         ret_code = LTTNG_ERR_UNK;
1023                         break;
1024                 }
1025                 goto end;
1026         }
1027
1028         DBG("Kernel track %s %d for session id %" PRIu64,
1029             lttng_process_attr_to_string(process_attr),
1030             integral_value,
1031             session->id);
1032         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1033                 /*
1034                  * Maintain a special case for the process ID process attribute
1035                  * tracker as it was the only supported attribute prior to 2.12.
1036                  */
1037                 ret = kernctl_track_pid(session->fd, integral_value);
1038         } else {
1039                 ret = kernctl_track_id(session->fd, process_attr, integral_value);
1040         }
1041         if (ret == 0) {
1042                 ret_code = LTTNG_OK;
1043                 goto end;
1044         }
1045
1046         kernel_wait_quiescent();
1047
1048         /* kern-ctl error handling */
1049         switch (-ret) {
1050         case 0:
1051                 ret_code = LTTNG_OK;
1052                 break;
1053         case EINVAL:
1054                 ret_code = LTTNG_ERR_INVALID;
1055                 break;
1056         case ENOMEM:
1057                 ret_code = LTTNG_ERR_NOMEM;
1058                 break;
1059         case EEXIST:
1060                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1061                 break;
1062         default:
1063                 ret_code = LTTNG_ERR_UNK;
1064                 break;
1065         }
1066
1067         /* Attempt to remove the value from the tracker. */
1068         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1069         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1070                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1071                     lttng_process_attr_to_string(process_attr),
1072                     integral_value);
1073         }
1074 end:
1075         return ret_code;
1076 }
1077
1078 enum lttng_error_code
1079 kernel_process_attr_tracker_inclusion_set_remove_value(struct ltt_kernel_session *session,
1080                                                        enum lttng_process_attr process_attr,
1081                                                        const struct process_attr_value *value)
1082 {
1083         int ret, integral_value;
1084         enum lttng_error_code ret_code;
1085         struct process_attr_tracker *tracker;
1086         enum process_attr_tracker_status status;
1087
1088         /*
1089          * Convert process attribute tracker value to the integral
1090          * representation required by the kern-ctl API.
1091          */
1092         switch (process_attr) {
1093         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1094         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1095                 integral_value = (int) value->value.pid;
1096                 break;
1097         case LTTNG_PROCESS_ATTR_USER_ID:
1098         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1099                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1100                         uid_t uid;
1101
1102                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1103                         if (ret_code != LTTNG_OK) {
1104                                 goto end;
1105                         }
1106                         integral_value = (int) uid;
1107                 } else {
1108                         integral_value = (int) value->value.uid;
1109                 }
1110                 break;
1111         case LTTNG_PROCESS_ATTR_GROUP_ID:
1112         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1113                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1114                         gid_t gid;
1115
1116                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1117                         if (ret_code != LTTNG_OK) {
1118                                 goto end;
1119                         }
1120                         integral_value = (int) gid;
1121                 } else {
1122                         integral_value = (int) value->value.gid;
1123                 }
1124                 break;
1125         default:
1126                 ret_code = LTTNG_ERR_INVALID;
1127                 goto end;
1128         }
1129
1130         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1131         if (!tracker) {
1132                 ret_code = LTTNG_ERR_INVALID;
1133                 goto end;
1134         }
1135
1136         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1137         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1138                 switch (status) {
1139                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1140                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1141                         break;
1142                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1143                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1144                         break;
1145                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1146                 default:
1147                         ret_code = LTTNG_ERR_UNK;
1148                         break;
1149                 }
1150                 goto end;
1151         }
1152
1153         DBG("Kernel track %s %d for session id %" PRIu64,
1154             lttng_process_attr_to_string(process_attr),
1155             integral_value,
1156             session->id);
1157         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1158                 /*
1159                  * Maintain a special case for the process ID process attribute
1160                  * tracker as it was the only supported attribute prior to 2.12.
1161                  */
1162                 ret = kernctl_untrack_pid(session->fd, integral_value);
1163         } else {
1164                 ret = kernctl_untrack_id(session->fd, process_attr, integral_value);
1165         }
1166         if (ret == 0) {
1167                 ret_code = LTTNG_OK;
1168                 goto end;
1169         }
1170         kernel_wait_quiescent();
1171
1172         /* kern-ctl error handling */
1173         switch (-ret) {
1174         case 0:
1175                 ret_code = LTTNG_OK;
1176                 break;
1177         case EINVAL:
1178                 ret_code = LTTNG_ERR_INVALID;
1179                 break;
1180         case ENOMEM:
1181                 ret_code = LTTNG_ERR_NOMEM;
1182                 break;
1183         case ENOENT:
1184                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1185                 break;
1186         default:
1187                 ret_code = LTTNG_ERR_UNK;
1188                 break;
1189         }
1190
1191         /* Attempt to add the value to the tracker. */
1192         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1193         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1194                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1195                     lttng_process_attr_to_string(process_attr),
1196                     integral_value);
1197         }
1198 end:
1199         return ret_code;
1200 }
1201
1202 /*
1203  * Create kernel metadata, open from the kernel tracer and add it to the
1204  * kernel session.
1205  */
1206 int kernel_open_metadata(struct ltt_kernel_session *session)
1207 {
1208         int ret;
1209         struct ltt_kernel_metadata *lkm = nullptr;
1210
1211         LTTNG_ASSERT(session);
1212
1213         /* Allocate kernel metadata */
1214         lkm = trace_kernel_create_metadata();
1215         if (lkm == nullptr) {
1216                 goto error;
1217         }
1218
1219         /* Kernel tracer metadata creation */
1220         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1221         if (ret < 0) {
1222                 goto error_open;
1223         }
1224
1225         lkm->fd = ret;
1226         lkm->key = ++next_kernel_channel_key;
1227         /* Prevent fd duplication after execlp() */
1228         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1229         if (ret < 0) {
1230                 PERROR("fcntl session fd");
1231         }
1232
1233         session->metadata = lkm;
1234
1235         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1236
1237         return 0;
1238
1239 error_open:
1240         trace_kernel_destroy_metadata(lkm);
1241 error:
1242         return -1;
1243 }
1244
1245 /*
1246  * Start tracing session.
1247  */
1248 int kernel_start_session(struct ltt_kernel_session *session)
1249 {
1250         int ret;
1251
1252         LTTNG_ASSERT(session);
1253
1254         ret = kernctl_start_session(session->fd);
1255         if (ret < 0) {
1256                 PERROR("ioctl start session");
1257                 goto error;
1258         }
1259
1260         DBG("Kernel session started");
1261
1262         return 0;
1263
1264 error:
1265         return ret;
1266 }
1267
1268 /*
1269  * Make a kernel wait to make sure in-flight probe have completed.
1270  */
1271 void kernel_wait_quiescent()
1272 {
1273         int ret;
1274         int fd = kernel_tracer_fd;
1275
1276         DBG("Kernel quiescent wait on %d", fd);
1277
1278         ret = kernctl_wait_quiescent(fd);
1279         if (ret < 0) {
1280                 PERROR("wait quiescent ioctl");
1281                 ERR("Kernel quiescent wait failed");
1282         }
1283 }
1284
1285 /*
1286  *  Force flush buffer of metadata.
1287  */
1288 int kernel_metadata_flush_buffer(int fd)
1289 {
1290         int ret;
1291
1292         DBG("Kernel flushing metadata buffer on fd %d", fd);
1293
1294         ret = kernctl_buffer_flush(fd);
1295         if (ret < 0) {
1296                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1297         }
1298
1299         return 0;
1300 }
1301
1302 /*
1303  * Force flush buffer for channel.
1304  */
1305 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1306 {
1307         int ret;
1308         struct ltt_kernel_stream *stream;
1309
1310         LTTNG_ASSERT(channel);
1311
1312         DBG("Flush buffer for channel %s", channel->channel->name);
1313
1314         cds_list_for_each_entry (stream, &channel->stream_list.head, list) {
1315                 DBG("Flushing channel stream %d", stream->fd);
1316                 ret = kernctl_buffer_flush(stream->fd);
1317                 if (ret < 0) {
1318                         PERROR("ioctl");
1319                         ERR("Fail to flush buffer for stream %d (ret: %d)", stream->fd, ret);
1320                 }
1321         }
1322
1323         return 0;
1324 }
1325
1326 /*
1327  * Stop tracing session.
1328  */
1329 int kernel_stop_session(struct ltt_kernel_session *session)
1330 {
1331         int ret;
1332
1333         LTTNG_ASSERT(session);
1334
1335         ret = kernctl_stop_session(session->fd);
1336         if (ret < 0) {
1337                 goto error;
1338         }
1339
1340         DBG("Kernel session stopped");
1341
1342         return 0;
1343
1344 error:
1345         return ret;
1346 }
1347
1348 /*
1349  * Open stream of channel, register it to the kernel tracer and add it
1350  * to the stream list of the channel.
1351  *
1352  * Note: given that the streams may appear in random order wrt CPU
1353  * number (e.g. cpu hotplug), the index value of the stream number in
1354  * the stream name is not necessarily linked to the CPU number.
1355  *
1356  * Return the number of created stream. Else, a negative value.
1357  */
1358 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1359 {
1360         int ret;
1361         struct ltt_kernel_stream *lks;
1362
1363         LTTNG_ASSERT(channel);
1364
1365         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1366                 lks = trace_kernel_create_stream(channel->channel->name, channel->stream_count);
1367                 if (lks == nullptr) {
1368                         ret = close(ret);
1369                         if (ret) {
1370                                 PERROR("close");
1371                         }
1372                         goto error;
1373                 }
1374
1375                 lks->fd = ret;
1376                 /* Prevent fd duplication after execlp() */
1377                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1378                 if (ret < 0) {
1379                         PERROR("fcntl session fd");
1380                 }
1381
1382                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1383                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1384
1385                 /* Add stream to channel stream list */
1386                 cds_list_add(&lks->list, &channel->stream_list.head);
1387                 channel->stream_count++;
1388
1389                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd, lks->state);
1390         }
1391
1392         return channel->stream_count;
1393
1394 error:
1395         return -1;
1396 }
1397
1398 /*
1399  * Open the metadata stream and set it to the kernel session.
1400  */
1401 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1402 {
1403         int ret;
1404
1405         LTTNG_ASSERT(session);
1406
1407         ret = kernctl_create_stream(session->metadata->fd);
1408         if (ret < 0) {
1409                 PERROR("kernel create metadata stream");
1410                 goto error;
1411         }
1412
1413         DBG("Kernel metadata stream created (fd: %d)", ret);
1414         session->metadata_stream_fd = ret;
1415         /* Prevent fd duplication after execlp() */
1416         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1417         if (ret < 0) {
1418                 PERROR("fcntl session fd");
1419         }
1420
1421         return 0;
1422
1423 error:
1424         return -1;
1425 }
1426
1427 /*
1428  * Get the event list from the kernel tracer and return the number of elements.
1429  */
1430 ssize_t kernel_list_events(struct lttng_event **events)
1431 {
1432         int fd, ret;
1433         char *event;
1434         size_t nbmem, count = 0;
1435         FILE *fp;
1436         struct lttng_event *elist;
1437
1438         LTTNG_ASSERT(events);
1439
1440         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1441         if (fd < 0) {
1442                 PERROR("kernel tracepoint list");
1443                 goto error;
1444         }
1445
1446         fp = fdopen(fd, "r");
1447         if (fp == nullptr) {
1448                 PERROR("kernel tracepoint list fdopen");
1449                 goto error_fp;
1450         }
1451
1452         /*
1453          * Init memory size counter
1454          * See kernel-ctl.h for explanation of this value
1455          */
1456         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1457         elist = calloc<lttng_event>(nbmem);
1458         if (elist == nullptr) {
1459                 PERROR("alloc list events");
1460                 count = -ENOMEM;
1461                 goto end;
1462         }
1463
1464         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1465                 if (count >= nbmem) {
1466                         struct lttng_event *new_elist;
1467                         size_t new_nbmem;
1468
1469                         new_nbmem = nbmem << 1;
1470                         DBG("Reallocating event list from %zu to %zu bytes", nbmem, new_nbmem);
1471                         new_elist = (lttng_event *) realloc(elist,
1472                                                             new_nbmem * sizeof(struct lttng_event));
1473                         if (new_elist == nullptr) {
1474                                 PERROR("realloc list events");
1475                                 free(event);
1476                                 free(elist);
1477                                 count = -ENOMEM;
1478                                 goto end;
1479                         }
1480                         /* Zero the new memory */
1481                         memset(new_elist + nbmem,
1482                                0,
1483                                (new_nbmem - nbmem) * sizeof(struct lttng_event));
1484                         nbmem = new_nbmem;
1485                         elist = new_elist;
1486                 }
1487                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1488                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1489                 elist[count].enabled = -1;
1490                 count++;
1491                 free(event);
1492         }
1493
1494         *events = elist;
1495         DBG("Kernel list events done (%zu events)", count);
1496 end:
1497         ret = fclose(fp); /* closes both fp and fd */
1498         if (ret) {
1499                 PERROR("fclose");
1500         }
1501         return count;
1502
1503 error_fp:
1504         ret = close(fd);
1505         if (ret) {
1506                 PERROR("close");
1507         }
1508 error:
1509         return -1;
1510 }
1511
1512 /*
1513  * Get kernel version and validate it.
1514  */
1515 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1516                             struct lttng_kernel_abi_tracer_abi_version *abi_version)
1517 {
1518         int ret;
1519
1520         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1521         if (ret < 0) {
1522                 ERR("Failed to retrieve the lttng-modules version");
1523                 goto error;
1524         }
1525
1526         /* Validate version */
1527         if (version->major != VERSION_MAJOR) {
1528                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1529                     version->major,
1530                     VERSION_MAJOR);
1531                 goto error_version;
1532         }
1533         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1534         if (ret < 0) {
1535                 ERR("Failed to retrieve lttng-modules ABI version");
1536                 goto error;
1537         }
1538         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1539                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1540                     abi_version->major,
1541                     abi_version->minor,
1542                     LTTNG_KERNEL_ABI_MAJOR_VERSION);
1543                 goto error;
1544         }
1545         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1546              version->major,
1547              version->minor,
1548              abi_version->major,
1549              abi_version->minor);
1550         return 0;
1551
1552 error_version:
1553         ret = -1;
1554
1555 error:
1556         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1557         return ret;
1558 }
1559
1560 /*
1561  * Kernel work-arounds called at the start of sessiond main().
1562  */
1563 int init_kernel_workarounds()
1564 {
1565         int ret;
1566         FILE *fp;
1567
1568         /*
1569          * boot_id needs to be read once before being used concurrently
1570          * to deal with a Linux kernel race. A fix is proposed for
1571          * upstream, but the work-around is needed for older kernels.
1572          */
1573         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1574         if (!fp) {
1575                 goto end_boot_id;
1576         }
1577         while (!feof(fp)) {
1578                 char buf[37] = "";
1579
1580                 ret = fread(buf, 1, sizeof(buf), fp);
1581                 if (ret < 0) {
1582                         /* Ignore error, we don't really care */
1583                 }
1584         }
1585         ret = fclose(fp);
1586         if (ret) {
1587                 PERROR("fclose");
1588         }
1589 end_boot_id:
1590         return 0;
1591 }
1592
1593 /*
1594  * Teardown of a kernel session, keeping data required by destroy notifiers.
1595  */
1596 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1597 {
1598         struct lttng_trace_chunk *trace_chunk;
1599
1600         if (ksess == nullptr) {
1601                 DBG3("No kernel session when tearing down session");
1602                 return;
1603         }
1604
1605         DBG("Tearing down kernel session");
1606         trace_chunk = ksess->current_trace_chunk;
1607
1608         /*
1609          * Destroy channels on the consumer if at least one FD has been sent and we
1610          * are in no output mode because the streams are in *no* monitor mode so we
1611          * have to send a command to clean them up or else they leaked.
1612          */
1613         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1614                 int ret;
1615                 struct consumer_socket *socket;
1616                 struct lttng_ht_iter iter;
1617
1618                 /* For each consumer socket. */
1619                 lttng::urcu::read_lock_guard read_lock;
1620
1621                 cds_lfht_for_each_entry (
1622                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1623                         struct ltt_kernel_channel *chan;
1624
1625                         /* For each channel, ask the consumer to destroy it. */
1626                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1627                                 ret = kernel_consumer_destroy_channel(socket, chan);
1628                                 if (ret < 0) {
1629                                         /* Consumer is probably dead. Use next socket. */
1630                                         continue;
1631                                 }
1632                         }
1633                 }
1634         }
1635
1636         /* Close any relayd session */
1637         consumer_output_send_destroy_relayd(ksess->consumer);
1638
1639         trace_kernel_destroy_session(ksess);
1640         lttng_trace_chunk_put(trace_chunk);
1641 }
1642
1643 /* Teardown of data required by destroy notifiers. */
1644 void kernel_free_session(struct ltt_kernel_session *ksess)
1645 {
1646         if (ksess == nullptr) {
1647                 return;
1648         }
1649         trace_kernel_free_session(ksess);
1650 }
1651
1652 /*
1653  * Destroy a kernel channel object. It does not do anything on the tracer side.
1654  */
1655 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1656 {
1657         struct ltt_kernel_session *ksess = nullptr;
1658
1659         LTTNG_ASSERT(kchan);
1660         LTTNG_ASSERT(kchan->channel);
1661
1662         DBG3("Kernel destroy channel %s", kchan->channel->name);
1663
1664         /* Update channel count of associated session. */
1665         if (kchan->session) {
1666                 /* Keep pointer reference so we can update it after the destroy. */
1667                 ksess = kchan->session;
1668         }
1669
1670         trace_kernel_destroy_channel(kchan);
1671
1672         /*
1673          * At this point the kernel channel is not visible anymore. This is safe
1674          * since in order to work on a visible kernel session, the tracing session
1675          * lock (ltt_session.lock) MUST be acquired.
1676          */
1677         if (ksess) {
1678                 ksess->channel_count--;
1679         }
1680 }
1681
1682 /*
1683  * Take a snapshot for a given kernel session.
1684  *
1685  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1686  */
1687 enum lttng_error_code kernel_snapshot_record(struct ltt_kernel_session *ksess,
1688                                              const struct consumer_output *output,
1689                                              uint64_t nb_packets_per_stream)
1690 {
1691         int err, ret, saved_metadata_fd;
1692         enum lttng_error_code status = LTTNG_OK;
1693         struct consumer_socket *socket;
1694         struct lttng_ht_iter iter;
1695         struct ltt_kernel_metadata *saved_metadata;
1696         char *trace_path = nullptr;
1697         size_t consumer_path_offset = 0;
1698
1699         LTTNG_ASSERT(ksess);
1700         LTTNG_ASSERT(ksess->consumer);
1701         LTTNG_ASSERT(output);
1702
1703         DBG("Kernel snapshot record started");
1704
1705         /* Save current metadata since the following calls will change it. */
1706         saved_metadata = ksess->metadata;
1707         saved_metadata_fd = ksess->metadata_stream_fd;
1708
1709         ret = kernel_open_metadata(ksess);
1710         if (ret < 0) {
1711                 status = LTTNG_ERR_KERN_META_FAIL;
1712                 goto error;
1713         }
1714
1715         ret = kernel_open_metadata_stream(ksess);
1716         if (ret < 0) {
1717                 status = LTTNG_ERR_KERN_META_FAIL;
1718                 goto error_open_stream;
1719         }
1720
1721         trace_path = setup_channel_trace_path(ksess->consumer, "", &consumer_path_offset);
1722         if (!trace_path) {
1723                 status = LTTNG_ERR_INVALID;
1724                 goto error;
1725         }
1726
1727         {
1728                 /* Send metadata to consumer and snapshot everything. */
1729                 lttng::urcu::read_lock_guard read_lock;
1730
1731                 cds_lfht_for_each_entry (output->socks->ht, &iter.iter, socket, node.node) {
1732                         struct ltt_kernel_channel *chan;
1733
1734                         pthread_mutex_lock(socket->lock);
1735                         /* This stream must not be monitored by the consumer. */
1736                         ret = kernel_consumer_add_metadata(socket, ksess, 0);
1737                         pthread_mutex_unlock(socket->lock);
1738                         if (ret < 0) {
1739                                 status = LTTNG_ERR_KERN_META_FAIL;
1740                                 goto error_consumer;
1741                         }
1742
1743                         /* For each channel, ask the consumer to snapshot it. */
1744                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1745                                 status =
1746                                         consumer_snapshot_channel(socket,
1747                                                                   chan->key,
1748                                                                   output,
1749                                                                   0,
1750                                                                   &trace_path[consumer_path_offset],
1751                                                                   nb_packets_per_stream);
1752                                 if (status != LTTNG_OK) {
1753                                         (void) kernel_consumer_destroy_metadata(socket,
1754                                                                                 ksess->metadata);
1755                                         goto error_consumer;
1756                                 }
1757                         }
1758
1759                         /* Snapshot metadata, */
1760                         status = consumer_snapshot_channel(socket,
1761                                                            ksess->metadata->key,
1762                                                            output,
1763                                                            1,
1764                                                            &trace_path[consumer_path_offset],
1765                                                            0);
1766                         if (status != LTTNG_OK) {
1767                                 goto error_consumer;
1768                         }
1769
1770                         /*
1771                          * The metadata snapshot is done, ask the consumer to destroy it since
1772                          * it's not monitored on the consumer side.
1773                          */
1774                         (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1775                 }
1776         }
1777
1778 error_consumer:
1779         /* Close newly opened metadata stream. It's now on the consumer side. */
1780         err = close(ksess->metadata_stream_fd);
1781         if (err < 0) {
1782                 PERROR("close snapshot kernel");
1783         }
1784
1785 error_open_stream:
1786         trace_kernel_destroy_metadata(ksess->metadata);
1787 error:
1788         /* Restore metadata state.*/
1789         ksess->metadata = saved_metadata;
1790         ksess->metadata_stream_fd = saved_metadata_fd;
1791         free(trace_path);
1792         return status;
1793 }
1794
1795 /*
1796  * Get the syscall mask array from the kernel tracer.
1797  *
1798  * Return 0 on success else a negative value. In both case, syscall_mask should
1799  * be freed.
1800  */
1801 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1802 {
1803         LTTNG_ASSERT(syscall_mask);
1804         LTTNG_ASSERT(nr_bits);
1805
1806         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1807 }
1808
1809 static int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1810 {
1811         int ret;
1812         struct lttng_kernel_abi_tracer_abi_version abi;
1813
1814         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1815         if (ret < 0) {
1816                 ERR("Failed to retrieve lttng-modules ABI version");
1817                 goto error;
1818         }
1819
1820         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1821 error:
1822         return ret;
1823 }
1824
1825 /*
1826  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1827  * version number.
1828  *
1829  * Return 1 on success, 0 when feature is not supported, negative value in case
1830  * of errors.
1831  */
1832 int kernel_supports_ring_buffer_snapshot_sample_positions()
1833 {
1834         /*
1835          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1836          */
1837         return kernel_tracer_abi_greater_or_equal(2, 3);
1838 }
1839
1840 /*
1841  * Check for the support of the packet sequence number via abi version number.
1842  *
1843  * Return 1 on success, 0 when feature is not supported, negative value in case
1844  * of errors.
1845  */
1846 int kernel_supports_ring_buffer_packet_sequence_number()
1847 {
1848         /*
1849          * Packet sequence number was introduced in LTTng 2.8,
1850          * lttng-modules ABI 2.1.
1851          */
1852         return kernel_tracer_abi_greater_or_equal(2, 1);
1853 }
1854
1855 /*
1856  * Check for the support of event notifiers via abi version number.
1857  *
1858  * Return 1 on success, 0 when feature is not supported, negative value in case
1859  * of errors.
1860  */
1861 int kernel_supports_event_notifiers()
1862 {
1863         /*
1864          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1865          */
1866         return kernel_tracer_abi_greater_or_equal(2, 6);
1867 }
1868
1869 /*
1870  * Rotate a kernel session.
1871  *
1872  * Return LTTNG_OK on success or else an LTTng error code.
1873  */
1874 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1875 {
1876         int ret;
1877         enum lttng_error_code status = LTTNG_OK;
1878         struct consumer_socket *socket;
1879         struct lttng_ht_iter iter;
1880         struct ltt_kernel_session *ksess = session->kernel_session;
1881
1882         LTTNG_ASSERT(ksess);
1883         LTTNG_ASSERT(ksess->consumer);
1884
1885         DBG("Rotate kernel session %s started (session %" PRIu64 ")", session->name, session->id);
1886
1887         {
1888                 /*
1889                  * Note that this loop will end after one iteration given that there is
1890                  * only one kernel consumer.
1891                  */
1892                 lttng::urcu::read_lock_guard read_lock;
1893
1894                 cds_lfht_for_each_entry (
1895                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1896                         struct ltt_kernel_channel *chan;
1897
1898                         /* For each channel, ask the consumer to rotate it. */
1899                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1900                                 DBG("Rotate kernel channel %" PRIu64 ", session %s",
1901                                     chan->key,
1902                                     session->name);
1903                                 ret = consumer_rotate_channel(socket,
1904                                                               chan->key,
1905                                                               ksess->consumer,
1906                                                               /* is_metadata_channel */ false);
1907                                 if (ret < 0) {
1908                                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1909                                         goto error;
1910                                 }
1911                         }
1912
1913                         /*
1914                          * Rotate the metadata channel.
1915                          */
1916                         ret = consumer_rotate_channel(socket,
1917                                                       ksess->metadata->key,
1918                                                       ksess->consumer,
1919                                                       /* is_metadata_channel */ true);
1920                         if (ret < 0) {
1921                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1922                                 goto error;
1923                         }
1924                 }
1925         }
1926
1927 error:
1928         return status;
1929 }
1930
1931 enum lttng_error_code kernel_create_channel_subdirectories(const struct ltt_kernel_session *ksess)
1932 {
1933         enum lttng_error_code ret = LTTNG_OK;
1934         enum lttng_trace_chunk_status chunk_status;
1935
1936         lttng::urcu::read_lock_guard read_lock;
1937         LTTNG_ASSERT(ksess->current_trace_chunk);
1938
1939         /*
1940          * Create the index subdirectory which will take care
1941          * of implicitly creating the channel's path.
1942          */
1943         chunk_status = lttng_trace_chunk_create_subdirectory(
1944                 ksess->current_trace_chunk, DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1945         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1946                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1947                 goto error;
1948         }
1949 error:
1950         return ret;
1951 }
1952
1953 /*
1954  * Setup necessary data for kernel tracer action.
1955  */
1956 int init_kernel_tracer()
1957 {
1958         int ret;
1959         bool is_root = !getuid();
1960
1961         /* Modprobe lttng kernel modules */
1962         ret = modprobe_lttng_control();
1963         if (ret < 0) {
1964                 goto error;
1965         }
1966
1967         /* Open debugfs lttng */
1968         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
1969         if (kernel_tracer_fd < 0) {
1970                 DBG("Failed to open %s", module_proc_lttng);
1971                 goto error_open;
1972         }
1973
1974         /* Validate kernel version */
1975         ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
1976         if (ret < 0) {
1977                 goto error_version;
1978         }
1979
1980         ret = modprobe_lttng_data();
1981         if (ret < 0) {
1982                 goto error_modules;
1983         }
1984
1985         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
1986         if (ret < 0) {
1987                 goto error_modules;
1988         }
1989         if (ret < 1) {
1990                 WARN("Kernel tracer does not support buffer monitoring. "
1991                      "The monitoring timer of channels in the kernel domain "
1992                      "will be set to 0 (disabled).");
1993         }
1994
1995         ret = kernel_supports_event_notifiers();
1996         if (ret < 0) {
1997                 ERR("Failed to check for kernel tracer event notifier support");
1998                 goto error_modules;
1999         }
2000         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2001         if (ret < 0) {
2002                 /* This is not fatal. */
2003                 WARN("Failed to create kernel event notifier group");
2004                 kernel_tracer_event_notifier_group_fd = -1;
2005         } else {
2006                 enum event_notifier_error_accounting_status error_accounting_status;
2007                 enum lttng_error_code error_code_ret =
2008                         kernel_create_event_notifier_group_notification_fd(
2009                                 &kernel_tracer_event_notifier_group_notification_fd);
2010
2011                 if (error_code_ret != LTTNG_OK) {
2012                         goto error_modules;
2013                 }
2014
2015                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2016                         kernel_tracer_event_notifier_group_fd);
2017                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2018                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2019                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2020                         goto error_modules;
2021                 }
2022
2023                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2024                         DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
2025                 if (!kernel_token_to_event_notifier_rule_ht) {
2026                         goto error_token_ht;
2027                 }
2028         }
2029
2030         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2031             kernel_tracer_fd,
2032             kernel_tracer_event_notifier_group_fd,
2033             kernel_tracer_event_notifier_group_notification_fd);
2034
2035         ret = syscall_init_table(kernel_tracer_fd);
2036         if (ret < 0) {
2037                 ERR("Unable to populate syscall table. Syscall tracing won't "
2038                     "work for this session daemon.");
2039         }
2040
2041         return 0;
2042
2043 error_version:
2044         modprobe_remove_lttng_control();
2045         ret = close(kernel_tracer_fd);
2046         if (ret) {
2047                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2048         }
2049
2050         kernel_tracer_fd = -1;
2051         return LTTNG_ERR_KERN_VERSION;
2052
2053 error_token_ht:
2054         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2055         if (ret) {
2056                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2057                        kernel_tracer_event_notifier_group_notification_fd);
2058         }
2059
2060         kernel_tracer_event_notifier_group_notification_fd = -1;
2061
2062 error_modules:
2063         ret = close(kernel_tracer_event_notifier_group_fd);
2064         if (ret) {
2065                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2066                        kernel_tracer_event_notifier_group_fd);
2067         }
2068
2069         kernel_tracer_event_notifier_group_fd = -1;
2070
2071         ret = close(kernel_tracer_fd);
2072         if (ret) {
2073                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2074         }
2075
2076         kernel_tracer_fd = -1;
2077
2078 error_open:
2079         modprobe_remove_lttng_control();
2080
2081 error:
2082         WARN("No kernel tracer available");
2083         kernel_tracer_fd = -1;
2084         if (!is_root) {
2085                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2086         } else {
2087                 return LTTNG_ERR_KERN_NA;
2088         }
2089 }
2090
2091 void cleanup_kernel_tracer()
2092 {
2093         DBG2("Closing kernel event notifier group notification file descriptor");
2094         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2095                 int ret = notification_thread_command_remove_tracer_event_source(
2096                         the_notification_thread_handle,
2097                         kernel_tracer_event_notifier_group_notification_fd);
2098                 if (ret != LTTNG_OK) {
2099                         ERR("Failed to remove kernel event notifier notification from notification thread");
2100                 }
2101
2102                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2103                 if (ret) {
2104                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2105                                kernel_tracer_event_notifier_group_notification_fd);
2106                 }
2107
2108                 kernel_tracer_event_notifier_group_notification_fd = -1;
2109         }
2110
2111         if (kernel_token_to_event_notifier_rule_ht) {
2112                 const int ret = cds_lfht_destroy(kernel_token_to_event_notifier_rule_ht, nullptr);
2113                 LTTNG_ASSERT(ret == 0);
2114         }
2115
2116         DBG2("Closing kernel event notifier group file descriptor");
2117         if (kernel_tracer_event_notifier_group_fd >= 0) {
2118                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2119
2120                 if (ret) {
2121                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2122                                kernel_tracer_event_notifier_group_fd);
2123                 }
2124
2125                 kernel_tracer_event_notifier_group_fd = -1;
2126         }
2127
2128         DBG2("Closing kernel fd");
2129         if (kernel_tracer_fd >= 0) {
2130                 const int ret = close(kernel_tracer_fd);
2131
2132                 if (ret) {
2133                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2134                                kernel_tracer_fd);
2135                 }
2136
2137                 kernel_tracer_fd = -1;
2138         }
2139
2140         free(syscall_table);
2141 }
2142
2143 bool kernel_tracer_is_initialized()
2144 {
2145         return kernel_tracer_fd >= 0;
2146 }
2147
2148 /*
2149  *  Clear a kernel session.
2150  *
2151  * Return LTTNG_OK on success or else an LTTng error code.
2152  */
2153 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2154 {
2155         int ret;
2156         enum lttng_error_code status = LTTNG_OK;
2157         struct consumer_socket *socket;
2158         struct lttng_ht_iter iter;
2159         struct ltt_kernel_session *ksess = session->kernel_session;
2160
2161         LTTNG_ASSERT(ksess);
2162         LTTNG_ASSERT(ksess->consumer);
2163
2164         DBG("Clear kernel session %s (session %" PRIu64 ")", session->name, session->id);
2165
2166         if (ksess->active) {
2167                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2168                 status = LTTNG_ERR_FATAL;
2169                 goto end;
2170         }
2171
2172         {
2173                 /*
2174                  * Note that this loop will end after one iteration given that there is
2175                  * only one kernel consumer.
2176                  */
2177                 lttng::urcu::read_lock_guard read_lock;
2178
2179                 cds_lfht_for_each_entry (
2180                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
2181                         struct ltt_kernel_channel *chan;
2182
2183                         /* For each channel, ask the consumer to clear it. */
2184                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
2185                                 DBG("Clear kernel channel %" PRIu64 ", session %s",
2186                                     chan->key,
2187                                     session->name);
2188                                 ret = consumer_clear_channel(socket, chan->key);
2189                                 if (ret < 0) {
2190                                         goto error;
2191                                 }
2192                         }
2193
2194                         if (!ksess->metadata) {
2195                                 /*
2196                                  * Nothing to do for the metadata.
2197                                  * This is a snapshot session.
2198                                  * The metadata is genererated on the fly.
2199                                  */
2200                                 continue;
2201                         }
2202
2203                         /*
2204                          * Clear the metadata channel.
2205                          * Metadata channel is not cleared per se but we still need to
2206                          * perform a rotation operation on it behind the scene.
2207                          */
2208                         ret = consumer_clear_channel(socket, ksess->metadata->key);
2209                         if (ret < 0) {
2210                                 goto error;
2211                         }
2212                 }
2213         }
2214
2215         goto end;
2216 error:
2217         switch (-ret) {
2218         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2219                 status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2220                 break;
2221         default:
2222                 status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2223                 break;
2224         }
2225 end:
2226         return status;
2227 }
2228
2229 enum lttng_error_code
2230 kernel_create_event_notifier_group_notification_fd(int *event_notifier_group_notification_fd)
2231 {
2232         int local_fd = -1, ret;
2233         enum lttng_error_code error_code_ret;
2234
2235         LTTNG_ASSERT(event_notifier_group_notification_fd);
2236
2237         ret = kernctl_create_event_notifier_group_notification_fd(
2238                 kernel_tracer_event_notifier_group_fd);
2239         if (ret < 0) {
2240                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2241                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2242                 goto error;
2243         }
2244
2245         local_fd = ret;
2246
2247         /* Prevent fd duplication after execlp(). */
2248         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2249         if (ret < 0) {
2250                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2251                        local_fd);
2252                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2253                 goto error;
2254         }
2255
2256         DBG("Created kernel notifier group notification file descriptor: fd = %d", local_fd);
2257         error_code_ret = LTTNG_OK;
2258         *event_notifier_group_notification_fd = local_fd;
2259         local_fd = -1;
2260
2261 error:
2262         if (local_fd >= 0) {
2263                 ret = close(local_fd);
2264                 if (ret) {
2265                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2266                                local_fd);
2267                 }
2268         }
2269
2270         return error_code_ret;
2271 }
2272
2273 enum lttng_error_code
2274 kernel_destroy_event_notifier_group_notification_fd(int event_notifier_group_notification_fd)
2275 {
2276         enum lttng_error_code ret_code = LTTNG_OK;
2277
2278         DBG("Closing event notifier group notification file descriptor: fd = %d",
2279             event_notifier_group_notification_fd);
2280         if (event_notifier_group_notification_fd >= 0) {
2281                 const int ret = close(event_notifier_group_notification_fd);
2282                 if (ret) {
2283                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2284                                event_notifier_group_notification_fd);
2285                 }
2286         }
2287
2288         return ret_code;
2289 }
2290
2291 static unsigned long hash_trigger(const struct lttng_trigger *trigger)
2292 {
2293         const struct lttng_condition *condition = lttng_trigger_get_const_condition(trigger);
2294
2295         return lttng_condition_hash(condition);
2296 }
2297
2298 static int match_trigger(struct cds_lfht_node *node, const void *key)
2299 {
2300         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2301         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2302
2303         event_notifier_rule =
2304                 caa_container_of(node, const struct ltt_kernel_event_notifier_rule, ht_node);
2305
2306         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2307 }
2308
2309 static enum lttng_error_code kernel_create_event_notifier_rule(
2310         struct lttng_trigger *trigger, const struct lttng_credentials *creds, uint64_t token)
2311 {
2312         int err, fd, ret = 0;
2313         enum lttng_error_code error_code_ret;
2314         enum lttng_condition_status condition_status;
2315         enum lttng_condition_type condition_type;
2316         enum lttng_event_rule_type event_rule_type;
2317         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2318         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2319         unsigned int capture_bytecode_count = 0, i;
2320         const struct lttng_condition *condition = nullptr;
2321         const struct lttng_event_rule *event_rule = nullptr;
2322         enum lttng_condition_status cond_status;
2323
2324         LTTNG_ASSERT(trigger);
2325
2326         condition = lttng_trigger_get_const_condition(trigger);
2327         LTTNG_ASSERT(condition);
2328
2329         condition_type = lttng_condition_get_type(condition);
2330         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2331
2332         /* Does not acquire a reference. */
2333         condition_status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2334         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2335         LTTNG_ASSERT(event_rule);
2336
2337         event_rule_type = lttng_event_rule_get_type(event_rule);
2338         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2339
2340         error_code_ret = trace_kernel_create_event_notifier_rule(
2341                 trigger,
2342                 token,
2343                 lttng_condition_event_rule_matches_get_error_counter_index(condition),
2344                 &event_notifier_rule);
2345         if (error_code_ret != LTTNG_OK) {
2346                 goto error;
2347         }
2348
2349         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(event_rule,
2350                                                                           &kernel_event_notifier);
2351         if (error_code_ret != LTTNG_OK) {
2352                 goto free_event;
2353         }
2354
2355         kernel_event_notifier.event.token = event_notifier_rule->token;
2356         kernel_event_notifier.error_counter_idx =
2357                 lttng_condition_event_rule_matches_get_error_counter_index(condition);
2358
2359         fd = kernctl_create_event_notifier(kernel_tracer_event_notifier_group_fd,
2360                                            &kernel_event_notifier);
2361         if (fd < 0) {
2362                 switch (-fd) {
2363                 case EEXIST:
2364                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2365                         break;
2366                 case ENOSYS:
2367                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2368                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2369                         break;
2370                 case ENOENT:
2371                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2372                              kernel_event_notifier.event.name);
2373                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2374                         break;
2375                 default:
2376                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2377                                fd,
2378                                kernel_event_notifier.event.name);
2379                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2380                 }
2381                 goto free_event;
2382         }
2383
2384         event_notifier_rule->fd = fd;
2385         /* Prevent fd duplication after execlp(). */
2386         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2387         if (err < 0) {
2388                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2389                        fd);
2390                 error_code_ret = LTTNG_ERR_FATAL;
2391                 goto set_cloexec_error;
2392         }
2393
2394         if (event_notifier_rule->filter) {
2395                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2396                 if (err < 0) {
2397                         switch (-err) {
2398                         case ENOMEM:
2399                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2400                                 break;
2401                         default:
2402                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2403                                 break;
2404                         }
2405                         goto filter_error;
2406                 }
2407         }
2408
2409         if (lttng_event_rule_get_type(event_rule) == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2410                 ret = userspace_probe_event_rule_add_callsites(
2411                         event_rule, creds, event_notifier_rule->fd);
2412                 if (ret) {
2413                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2414                         goto add_callsite_error;
2415                 }
2416         }
2417
2418         /* Set the capture bytecode if any. */
2419         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2420                 condition, &capture_bytecode_count);
2421         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2422
2423         for (i = 0; i < capture_bytecode_count; i++) {
2424                 const struct lttng_bytecode *capture_bytecode =
2425                         lttng_condition_event_rule_matches_get_capture_bytecode_at_index(condition,
2426                                                                                          i);
2427
2428                 if (capture_bytecode == nullptr) {
2429                         ERR("Unexpected NULL capture bytecode on condition");
2430                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2431                         goto capture_error;
2432                 }
2433
2434                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2435                 if (ret < 0) {
2436                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2437                             event_notifier_rule->fd);
2438                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2439                         goto capture_error;
2440                 }
2441         }
2442
2443         err = kernctl_enable(event_notifier_rule->fd);
2444         if (err < 0) {
2445                 switch (-err) {
2446                 case EEXIST:
2447                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2448                         break;
2449                 default:
2450                         PERROR("enable kernel event notifier");
2451                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2452                         break;
2453                 }
2454                 goto enable_error;
2455         }
2456
2457         /* Add trigger to kernel token mapping in the hash table. */
2458         {
2459                 lttng::urcu::read_lock_guard read_lock;
2460                 cds_lfht_add(kernel_token_to_event_notifier_rule_ht,
2461                              hash_trigger(trigger),
2462                              &event_notifier_rule->ht_node);
2463         }
2464
2465         DBG("Created kernel event notifier: name = '%s', fd = %d",
2466             kernel_event_notifier.event.name,
2467             event_notifier_rule->fd);
2468
2469         return LTTNG_OK;
2470
2471 capture_error:
2472 add_callsite_error:
2473 enable_error:
2474 set_cloexec_error:
2475 filter_error:
2476 {
2477         const int close_ret = close(event_notifier_rule->fd);
2478
2479         if (close_ret) {
2480                 PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2481                        event_notifier_rule->fd);
2482         }
2483 }
2484 free_event:
2485         free(event_notifier_rule);
2486 error:
2487         return error_code_ret;
2488 }
2489
2490 enum lttng_error_code kernel_register_event_notifier(struct lttng_trigger *trigger,
2491                                                      const struct lttng_credentials *cmd_creds)
2492 {
2493         enum lttng_error_code ret;
2494         enum lttng_condition_status status;
2495         enum lttng_domain_type domain_type;
2496         const struct lttng_event_rule *event_rule;
2497         const struct lttng_condition *const condition = lttng_trigger_get_const_condition(trigger);
2498         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2499
2500         LTTNG_ASSERT(condition);
2501
2502         /* Does not acquire a reference to the event rule. */
2503         status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2504         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2505
2506         domain_type = lttng_event_rule_get_domain_type(event_rule);
2507         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2508
2509         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2510         if (ret != LTTNG_OK) {
2511                 ERR("Failed to create kernel event notifier rule");
2512         }
2513
2514         return ret;
2515 }
2516
2517 enum lttng_error_code kernel_unregister_event_notifier(const struct lttng_trigger *trigger)
2518 {
2519         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2520         struct cds_lfht_node *node;
2521         struct cds_lfht_iter iter;
2522         enum lttng_error_code error_code_ret;
2523         int ret;
2524
2525         lttng::urcu::read_lock_guard read_lock;
2526
2527         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2528                         hash_trigger(trigger),
2529                         match_trigger,
2530                         trigger,
2531                         &iter);
2532
2533         node = cds_lfht_iter_get_node(&iter);
2534         if (!node) {
2535                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2536                 goto error;
2537         }
2538
2539         token_event_rule_element =
2540                 caa_container_of(node, struct ltt_kernel_event_notifier_rule, ht_node);
2541
2542         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2543         if (ret) {
2544                 error_code_ret = LTTNG_ERR_FATAL;
2545                 goto error;
2546         }
2547
2548         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2549         error_code_ret = LTTNG_OK;
2550
2551 error:
2552
2553         return error_code_ret;
2554 }
2555
2556 int kernel_get_notification_fd()
2557 {
2558         return kernel_tracer_event_notifier_group_notification_fd;
2559 }