src/common/fd-tracker/fd-tracker.cpp

   1 /*
   2  * Copyright (C) 2018-2020 Jérémie Galarneau <jeremie.galarneau@efficios.com>
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #include "fd-tracker.hpp"
   9 #include "inode.hpp"
  10
  11 #include <common/defaults.hpp>
  12 #include <common/error.hpp>
  13 #include <common/fs-handle-internal.hpp>
  14 #include <common/hashtable/hashtable.hpp>
  15 #include <common/hashtable/utils.hpp>
  16 #include <common/macros.hpp>
  17 #include <common/optional.hpp>
  18
  19 #include <fcntl.h>
  20 #include <inttypes.h>
  21 #include <pthread.h>
  22 #include <stdbool.h>
  23 #include <sys/stat.h>
  24 #include <sys/types.h>
  25 #include <urcu.h>
  26 #include <urcu/list.h>
  27 #include <urcu/rculfhash.h>
  28
  29 /* Tracker lock must be taken by the user. */
  30 #define TRACKED_COUNT(tracker)                                                          \
  31         ((tracker)->count.suspendable.active + (tracker)->count.suspendable.suspended + \
  32          (tracker)->count.unsuspendable)
  33
  34 /* Tracker lock must be taken by the user. */
  35 #define ACTIVE_COUNT(tracker) ((tracker)->count.suspendable.active + (tracker)->count.unsuspendable)
  36
  37 /* Tracker lock must be taken by the user. */
  38 #define SUSPENDED_COUNT(tracker) ((tracker)->count.suspendable.suspended)
  39
  40 /* Tracker lock must be taken by the user. */
  41 #define SUSPENDABLE_COUNT(tracker) \
  42         ((tracker)->count.suspendable.active + (tracker)->count.suspendable.suspended)
  43
  44 /* Tracker lock must be taken by the user. */
  45 #define UNSUSPENDABLE_COUNT(tracker) ((tracker)->count.unsuspendable)
  46
  47 struct fd_tracker {
  48         pthread_mutex_t lock;
  49         struct {
  50                 struct {
  51                         unsigned int active;
  52                         unsigned int suspended;
  53                 } suspendable;
  54                 unsigned int unsuspendable;
  55         } count;
  56         unsigned int capacity;
  57         struct {
  58                 uint64_t uses;
  59                 uint64_t misses;
  60                 /* Failures to suspend or restore fs handles. */
  61                 uint64_t errors;
  62         } stats;
  63         /*
  64          * The head of the active_handles list is always the least recently
  65          * used active handle. When an handle is used, it is removed from the
  66          * list and added to the end. When a file has to be suspended, the
  67          * first element in the list is "popped", suspended, and added to the
  68          * list of suspended handles.
  69          */
  70         struct cds_list_head active_handles;
  71         struct cds_list_head suspended_handles;
  72         struct cds_lfht *unsuspendable_fds;
  73         struct lttng_inode_registry *inode_registry;
  74         /* Unlinked files are moved in this directory under a unique name. */
  75         struct lttng_directory_handle *unlink_directory_handle;
  76         struct lttng_unlinked_file_pool *unlinked_file_pool;
  77 };
  78
  79 namespace {
  80 struct open_properties {
  81         int flags;
  82         LTTNG_OPTIONAL(mode_t) mode;
  83 };
  84
  85 /*
  86  * A fs_handle_tracked is not ref-counted. Therefore, it is assumed that a
  87  * handle is never in-use while it is being reclaimed. It can be
  88  * shared by multiple threads, but external synchronization is required
  89  * to ensure it is not still being used when it is reclaimed (close method).
  90  * In this respect, it is not different from a regular file descriptor.
  91  *
  92  * The fs_handle lock always nests _within_ the tracker's lock.
  93  */
  94 struct fs_handle_tracked {
  95         struct fs_handle parent;
  96         pthread_mutex_t lock;
  97         /*
  98          * Weak reference to the tracker. All fs_handles are assumed to have
  99          * been closed at the moment of the destruction of the fd_tracker.
 100          */
 101         struct fd_tracker *tracker;
 102         struct open_properties properties;
 103         struct lttng_inode *inode;
 104         int fd;
 105         /* inode number of the file at the time of the handle's creation. */
 106         uint64_t ino;
 107         bool in_use;
 108         /* Offset to which the file should be restored. */
 109         off_t offset;
 110         struct cds_list_head handles_list_node;
 111 };
 112
 113 struct unsuspendable_fd {
 114         /*
 115          * Accesses are only performed through the tracker, which is protected
 116          * by its own lock.
 117          */
 118         int fd;
 119         char *name;
 120         struct cds_lfht_node tracker_node;
 121         struct rcu_head rcu_head;
 122 };
 123
 124 struct {
 125         pthread_mutex_t lock;
 126         bool initialized;
 127         unsigned long value;
 128 } seed = {
 129         .lock = PTHREAD_MUTEX_INITIALIZER,
 130         .initialized = false,
 131         .value = 0,
 132 };
 133 } /* namespace */
 134
 135 static int match_fd(struct cds_lfht_node *node, const void *key);
 136 static void unsuspendable_fd_destroy(struct unsuspendable_fd *entry);
 137 static struct unsuspendable_fd *unsuspendable_fd_create(const char *name, int fd);
 138 static int open_from_properties(const struct lttng_directory_handle *dir_handle,
 139                                 const char *path,
 140                                 struct open_properties *properties);
 141
 142 static void fs_handle_tracked_log(struct fs_handle_tracked *handle);
 143 static int fs_handle_tracked_suspend(struct fs_handle_tracked *handle);
 144 static int fs_handle_tracked_restore(struct fs_handle_tracked *handle);
 145 static int fs_handle_tracked_get_fd(struct fs_handle *_handle);
 146 static void fs_handle_tracked_put_fd(struct fs_handle *_handle);
 147 static int fs_handle_tracked_unlink(struct fs_handle *_handle);
 148 static int fs_handle_tracked_close(struct fs_handle *_handle);
 149
 150 static void fd_tracker_track(struct fd_tracker *tracker, struct fs_handle_tracked *handle);
 151 static void fd_tracker_untrack(struct fd_tracker *tracker, struct fs_handle_tracked *handle);
 152 static int fd_tracker_suspend_handles(struct fd_tracker *tracker, unsigned int count);
 153 static int fd_tracker_restore_handle(struct fd_tracker *tracker, struct fs_handle_tracked *handle);
 154
 155 /* Match function of the tracker's unsuspendable_fds hash table. */
 156 static int match_fd(struct cds_lfht_node *node, const void *key)
 157 {
 158         struct unsuspendable_fd *entry =
 159                 lttng::utils::container_of(node, &unsuspendable_fd::tracker_node);
 160
 161         return hash_match_key_ulong((void *) (unsigned long) entry->fd, (void *) key);
 162 }
 163
 164 static void delete_unsuspendable_fd(struct rcu_head *head)
 165 {
 166         struct unsuspendable_fd *fd = caa_container_of(head, struct unsuspendable_fd, rcu_head);
 167
 168         free(fd->name);
 169         free(fd);
 170 }
 171
 172 static void unsuspendable_fd_destroy(struct unsuspendable_fd *entry)
 173 {
 174         if (!entry) {
 175                 return;
 176         }
 177         call_rcu(&entry->rcu_head, delete_unsuspendable_fd);
 178 }
 179
 180 static struct unsuspendable_fd *unsuspendable_fd_create(const char *name, int fd)
 181 {
 182         struct unsuspendable_fd *entry = zmalloc<unsuspendable_fd>();
 183
 184         if (!entry) {
 185                 goto error;
 186         }
 187         if (name) {
 188                 entry->name = strdup(name);
 189                 if (!entry->name) {
 190                         goto error;
 191                 }
 192         }
 193         cds_lfht_node_init(&entry->tracker_node);
 194         entry->fd = fd;
 195         return entry;
 196 error:
 197         unsuspendable_fd_destroy(entry);
 198         return nullptr;
 199 }
 200
 201 static void fs_handle_tracked_log(struct fs_handle_tracked *handle)
 202 {
 203         const char *path;
 204
 205         pthread_mutex_lock(&handle->lock);
 206         lttng_inode_borrow_location(handle->inode, nullptr, &path);
 207
 208         if (handle->fd >= 0) {
 209                 DBG_NO_LOC("    %s [active, fd %d%s]",
 210                            path,
 211                            handle->fd,
 212                            handle->in_use ? ", in use" : "");
 213         } else {
 214                 DBG_NO_LOC("    %s [suspended]", path);
 215         }
 216         pthread_mutex_unlock(&handle->lock);
 217 }
 218
 219 /* Tracker lock must be held by the caller. */
 220 static int fs_handle_tracked_suspend(struct fs_handle_tracked *handle)
 221 {
 222         int ret = 0;
 223         struct stat fs_stat;
 224         const char *path;
 225         const struct lttng_directory_handle *node_directory_handle;
 226
 227         pthread_mutex_lock(&handle->lock);
 228         lttng_inode_borrow_location(handle->inode, &node_directory_handle, &path);
 229         LTTNG_ASSERT(handle->fd >= 0);
 230         if (handle->in_use) {
 231                 /* This handle can't be suspended as it is currently in use. */
 232                 ret = -EAGAIN;
 233                 goto end;
 234         }
 235
 236         ret = lttng_directory_handle_stat(node_directory_handle, path, &fs_stat);
 237         if (ret) {
 238                 PERROR("Filesystem handle to %s cannot be suspended as stat() failed", path);
 239                 ret = -errno;
 240                 goto end;
 241         }
 242
 243         if (fs_stat.st_ino != handle->ino) {
 244                 /* Don't suspend as the handle would not be restorable. */
 245                 WARN("Filesystem handle to %s cannot be suspended as its inode changed", path);
 246                 ret = -ENOENT;
 247                 goto end;
 248         }
 249
 250         handle->offset = lseek(handle->fd, 0, SEEK_CUR);
 251         if (handle->offset == -1) {
 252                 WARN("Filesystem handle to %s cannot be suspended as lseek() failed to sample its current position",
 253                      path);
 254                 ret = -errno;
 255                 goto end;
 256         }
 257
 258         ret = close(handle->fd);
 259         if (ret) {
 260                 PERROR("Filesystem handle to %s cannot be suspended as close() failed", path);
 261                 ret = -errno;
 262                 goto end;
 263         }
 264         DBG("Suspended filesystem handle to %s (fd %i) at position %" PRId64,
 265             path,
 266             handle->fd,
 267             handle->offset);
 268         handle->fd = -1;
 269 end:
 270         if (ret) {
 271                 handle->tracker->stats.errors++;
 272         }
 273         pthread_mutex_unlock(&handle->lock);
 274         return ret;
 275 }
 276
 277 /* Caller must hold the tracker and handle's locks. */
 278 static int fs_handle_tracked_restore(struct fs_handle_tracked *handle)
 279 {
 280         int ret, fd = -1;
 281         const char *path;
 282         const struct lttng_directory_handle *node_directory_handle;
 283
 284         lttng_inode_borrow_location(handle->inode, &node_directory_handle, &path);
 285
 286         LTTNG_ASSERT(handle->fd == -1);
 287         LTTNG_ASSERT(path);
 288         ret = open_from_properties(node_directory_handle, path, &handle->properties);
 289         if (ret < 0) {
 290                 PERROR("Failed to restore filesystem handle to %s, open() failed", path);
 291                 ret = -errno;
 292                 goto end;
 293         }
 294         fd = ret;
 295
 296         ret = lseek(fd, handle->offset, SEEK_SET);
 297         if (ret < 0) {
 298                 PERROR("Failed to restore filesystem handle to %s, lseek() failed", path);
 299                 ret = -errno;
 300                 goto end;
 301         }
 302         DBG("Restored filesystem handle to %s (fd %i) at position %" PRId64,
 303             path,
 304             fd,
 305             handle->offset);
 306         ret = 0;
 307         handle->fd = fd;
 308         fd = -1;
 309 end:
 310         if (fd >= 0) {
 311                 (void) close(fd);
 312         }
 313         return ret;
 314 }
 315
 316 static int open_from_properties(const struct lttng_directory_handle *dir_handle,
 317                                 const char *path,
 318                                 struct open_properties *properties)
 319 {
 320         int ret;
 321
 322         /*
 323          * open() ignores the 'flags' parameter unless the O_CREAT or O_TMPFILE
 324          * flags are set. O_TMPFILE would not make sense in the context of a
 325          * suspendable fs_handle as it would not be restorable (see OPEN(2)),
 326          * thus it is ignored here.
 327          */
 328         if ((properties->flags & O_CREAT) && properties->mode.is_set) {
 329                 ret = lttng_directory_handle_open_file(
 330                         dir_handle, path, properties->flags, properties->mode.value);
 331         } else {
 332                 ret = lttng_directory_handle_open_file(dir_handle, path, properties->flags, 0);
 333         }
 334         /*
 335          * Some flags should not be used beyond the initial open() of a
 336          * restorable file system handle. O_CREAT and O_TRUNC must
 337          * be cleared since it would be unexpected to re-use them
 338          * when the handle is retored:
 339          *  - O_CREAT should not be needed as the file has been created
 340          *    on the initial call to open(),
 341          *  - O_TRUNC would destroy the file's contents by truncating it
 342          *    to length 0.
 343          */
 344         properties->flags &= ~(O_CREAT | O_TRUNC);
 345         if (ret < 0) {
 346                 ret = -errno;
 347                 goto end;
 348         }
 349 end:
 350         return ret;
 351 }
 352
 353 struct fd_tracker *fd_tracker_create(const char *unlinked_file_path, unsigned int capacity)
 354 {
 355         struct fd_tracker *tracker = zmalloc<fd_tracker>();
 356
 357         if (!tracker) {
 358                 goto end;
 359         }
 360
 361         pthread_mutex_lock(&seed.lock);
 362         if (!seed.initialized) {
 363                 seed.value = (unsigned long) time(nullptr);
 364                 seed.initialized = true;
 365         }
 366         pthread_mutex_unlock(&seed.lock);
 367
 368         CDS_INIT_LIST_HEAD(&tracker->active_handles);
 369         CDS_INIT_LIST_HEAD(&tracker->suspended_handles);
 370         tracker->capacity = capacity;
 371         tracker->unsuspendable_fds = cds_lfht_new(
 372                 DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
 373         if (!tracker->unsuspendable_fds) {
 374                 ERR("Failed to create fd-tracker's unsuspendable_fds hash table");
 375                 goto error;
 376         }
 377         tracker->inode_registry = lttng_inode_registry_create();
 378         if (!tracker->inode_registry) {
 379                 ERR("Failed to create fd-tracker's inode registry");
 380                 goto error;
 381         }
 382
 383         tracker->unlinked_file_pool = lttng_unlinked_file_pool_create(unlinked_file_path);
 384         if (!tracker->unlinked_file_pool) {
 385                 goto error;
 386         }
 387         DBG("File descriptor tracker created with a limit of %u simultaneously-opened FDs",
 388             capacity);
 389 end:
 390         return tracker;
 391 error:
 392         fd_tracker_destroy(tracker);
 393         return nullptr;
 394 }
 395
 396 void fd_tracker_log(struct fd_tracker *tracker)
 397 {
 398         struct fs_handle_tracked *handle;
 399         struct unsuspendable_fd *unsuspendable_fd;
 400         struct cds_lfht_iter iter;
 401
 402         pthread_mutex_lock(&tracker->lock);
 403         DBG_NO_LOC("File descriptor tracker");
 404         DBG_NO_LOC("  Stats:");
 405         DBG_NO_LOC("    uses:            %" PRIu64, tracker->stats.uses);
 406         DBG_NO_LOC("    misses:          %" PRIu64, tracker->stats.misses);
 407         DBG_NO_LOC("    errors:          %" PRIu64, tracker->stats.errors);
 408         DBG_NO_LOC("  Tracked:           %u", TRACKED_COUNT(tracker));
 409         DBG_NO_LOC("    active:          %u", ACTIVE_COUNT(tracker));
 410         DBG_NO_LOC("      suspendable:   %u", SUSPENDABLE_COUNT(tracker));
 411         DBG_NO_LOC("      unsuspendable: %u", UNSUSPENDABLE_COUNT(tracker));
 412         DBG_NO_LOC("    suspended:       %u", SUSPENDED_COUNT(tracker));
 413         DBG_NO_LOC("    capacity:        %u", tracker->capacity);
 414
 415         DBG_NO_LOC("  Tracked suspendable file descriptors");
 416         cds_list_for_each_entry (handle, &tracker->active_handles, handles_list_node) {
 417                 fs_handle_tracked_log(handle);
 418         }
 419         cds_list_for_each_entry (handle, &tracker->suspended_handles, handles_list_node) {
 420                 fs_handle_tracked_log(handle);
 421         }
 422         if (!SUSPENDABLE_COUNT(tracker)) {
 423                 DBG_NO_LOC("    None");
 424         }
 425
 426         DBG_NO_LOC("  Tracked unsuspendable file descriptors");
 427         rcu_read_lock();
 428         cds_lfht_for_each_entry (
 429                 tracker->unsuspendable_fds, &iter, unsuspendable_fd, tracker_node) {
 430                 DBG_NO_LOC("    %s [active, fd %d]",
 431                            unsuspendable_fd->name ?: "Unnamed",
 432                            unsuspendable_fd->fd);
 433         }
 434         rcu_read_unlock();
 435         if (!UNSUSPENDABLE_COUNT(tracker)) {
 436                 DBG_NO_LOC("    None");
 437         }
 438
 439         pthread_mutex_unlock(&tracker->lock);
 440 }
 441
 442 int fd_tracker_destroy(struct fd_tracker *tracker)
 443 {
 444         int ret = 0;
 445
 446         if (!tracker) {
 447                 goto end;
 448         }
 449         /*
 450          * Refuse to destroy the tracker as fs_handles may still old
 451          * weak references to the tracker.
 452          */
 453         pthread_mutex_lock(&tracker->lock);
 454         if (TRACKED_COUNT(tracker)) {
 455                 ERR("A file descriptor leak has been detected: %u tracked file descriptors are still being tracked",
 456                     TRACKED_COUNT(tracker));
 457                 pthread_mutex_unlock(&tracker->lock);
 458                 fd_tracker_log(tracker);
 459                 ret = -1;
 460                 goto end;
 461         }
 462         pthread_mutex_unlock(&tracker->lock);
 463
 464         if (tracker->unsuspendable_fds) {
 465                 ret = cds_lfht_destroy(tracker->unsuspendable_fds, nullptr);
 466                 LTTNG_ASSERT(!ret);
 467         }
 468
 469         lttng_inode_registry_destroy(tracker->inode_registry);
 470         lttng_unlinked_file_pool_destroy(tracker->unlinked_file_pool);
 471         pthread_mutex_destroy(&tracker->lock);
 472         free(tracker);
 473 end:
 474         return ret;
 475 }
 476
 477 struct fs_handle *fd_tracker_open_fs_handle(struct fd_tracker *tracker,
 478                                             struct lttng_directory_handle *directory,
 479                                             const char *path,
 480                                             int flags,
 481                                             mode_t *mode)
 482 {
 483         int ret;
 484         struct fs_handle_tracked *handle = nullptr;
 485         struct stat fd_stat;
 486         struct open_properties properties = { .flags = flags,
 487                                               .mode = {
 488                                                       .is_set = !!mode,
 489                                                       .value =
 490                                                               static_cast<mode_t>(mode ? *mode : 0),
 491                                               } };
 492
 493         pthread_mutex_lock(&tracker->lock);
 494         if (ACTIVE_COUNT(tracker) == tracker->capacity) {
 495                 if (tracker->count.suspendable.active > 0) {
 496                         ret = fd_tracker_suspend_handles(tracker, 1);
 497                         if (ret) {
 498                                 goto end;
 499                         }
 500                 } else {
 501                         /*
 502                          * There are not enough active suspendable file
 503                          * descriptors to open a new fd and still accommodate
 504                          * the tracker's capacity.
 505                          */
 506                         WARN("Cannot open file system handle, too many unsuspendable file descriptors are opened (%u)",
 507                              tracker->count.unsuspendable);
 508                         goto end;
 509                 }
 510         }
 511
 512         handle = zmalloc<fs_handle_tracked>();
 513         if (!handle) {
 514                 goto end;
 515         }
 516         handle->parent = (typeof(handle->parent)){
 517                 .get_fd = fs_handle_tracked_get_fd,
 518                 .put_fd = fs_handle_tracked_put_fd,
 519                 .unlink = fs_handle_tracked_unlink,
 520                 .close = fs_handle_tracked_close,
 521         };
 522
 523         handle->tracker = tracker;
 524
 525         ret = pthread_mutex_init(&handle->lock, nullptr);
 526         if (ret) {
 527                 PERROR("Failed to initialize handle mutex while creating fs handle");
 528                 goto error_mutex_init;
 529         }
 530
 531         handle->fd = open_from_properties(directory, path, &properties);
 532         if (handle->fd < 0) {
 533                 PERROR("Failed to open fs handle to %s, open() returned", path);
 534                 goto error;
 535         }
 536
 537         handle->properties = properties;
 538
 539         handle->inode = lttng_inode_registry_get_inode(
 540                 tracker->inode_registry, directory, path, handle->fd, tracker->unlinked_file_pool);
 541         if (!handle->inode) {
 542                 ERR("Failed to get lttng_inode corresponding to file %s", path);
 543                 goto error;
 544         }
 545
 546         if (fstat(handle->fd, &fd_stat)) {
 547                 PERROR("Failed to retrieve file descriptor inode while creating fs handle, fstat() returned");
 548                 goto error;
 549         }
 550         handle->ino = fd_stat.st_ino;
 551
 552         fd_tracker_track(tracker, handle);
 553 end:
 554         pthread_mutex_unlock(&tracker->lock);
 555         return handle ? &handle->parent : nullptr;
 556 error:
 557         if (handle->inode) {
 558                 lttng_inode_put(handle->inode);
 559         }
 560         pthread_mutex_destroy(&handle->lock);
 561 error_mutex_init:
 562         free(handle);
 563         handle = nullptr;
 564         goto end;
 565 }
 566
 567 /* Caller must hold the tracker's lock. */
 568 static int fd_tracker_suspend_handles(struct fd_tracker *tracker, unsigned int count)
 569 {
 570         unsigned int left_to_close = count;
 571         unsigned int attempts_left = tracker->count.suspendable.active;
 572         struct fs_handle_tracked *handle, *tmp;
 573
 574         cds_list_for_each_entry_safe (handle, tmp, &tracker->active_handles, handles_list_node) {
 575                 int ret;
 576
 577                 fd_tracker_untrack(tracker, handle);
 578                 ret = fs_handle_tracked_suspend(handle);
 579                 fd_tracker_track(tracker, handle);
 580                 if (!ret) {
 581                         left_to_close--;
 582                 }
 583                 attempts_left--;
 584
 585                 if (left_to_close == 0 || attempts_left == 0) {
 586                         break;
 587                 }
 588         }
 589         return left_to_close ? -EMFILE : 0;
 590 }
 591
 592 int fd_tracker_open_unsuspendable_fd(struct fd_tracker *tracker,
 593                                      int *out_fds,
 594                                      const char **names,
 595                                      unsigned int fd_count,
 596                                      fd_open_cb open,
 597                                      void *user_data)
 598 {
 599         int ret, user_ret, i, fds_to_suspend;
 600         unsigned int active_fds;
 601         struct unsuspendable_fd **entries;
 602
 603         entries = calloc<unsuspendable_fd *>(fd_count);
 604         if (!entries) {
 605                 ret = -1;
 606                 goto end;
 607         }
 608
 609         pthread_mutex_lock(&tracker->lock);
 610
 611         active_fds = ACTIVE_COUNT(tracker);
 612         fds_to_suspend = (int) active_fds + (int) fd_count - (int) tracker->capacity;
 613         if (fds_to_suspend > 0) {
 614                 if (fds_to_suspend <= tracker->count.suspendable.active) {
 615                         ret = fd_tracker_suspend_handles(tracker, fds_to_suspend);
 616                         if (ret) {
 617                                 goto end_unlock;
 618                         }
 619                 } else {
 620                         /*
 621                          * There are not enough active suspendable file
 622                          * descriptors to open a new fd and still accommodates the
 623                          * tracker's capacity.
 624                          */
 625                         WARN("Cannot open unsuspendable fd, too many unsuspendable file descriptors are opened (%u)",
 626                              tracker->count.unsuspendable);
 627                         ret = -EMFILE;
 628                         goto end_unlock;
 629                 }
 630         }
 631
 632         user_ret = open(user_data, out_fds);
 633         if (user_ret) {
 634                 ret = user_ret;
 635                 goto end_unlock;
 636         }
 637
 638         /*
 639          * Add the fds returned by the user's callback to the hashtable
 640          * of unsuspendable fds.
 641          */
 642         for (i = 0; i < fd_count; i++) {
 643                 struct unsuspendable_fd *entry =
 644                         unsuspendable_fd_create(names ? names[i] : nullptr, out_fds[i]);
 645
 646                 if (!entry) {
 647                         ret = -1;
 648                         goto end_free_entries;
 649                 }
 650                 entries[i] = entry;
 651         }
 652
 653         rcu_read_lock();
 654         for (i = 0; i < fd_count; i++) {
 655                 struct cds_lfht_node *node;
 656                 struct unsuspendable_fd *entry = entries[i];
 657
 658                 node = cds_lfht_add_unique(tracker->unsuspendable_fds,
 659                                            hash_key_ulong((void *) (unsigned long) out_fds[i],
 660                                                           seed.value),
 661                                            match_fd,
 662                                            (void *) (unsigned long) out_fds[i],
 663                                            &entry->tracker_node);
 664
 665                 if (node != &entry->tracker_node) {
 666                         ret = -EEXIST;
 667                         rcu_read_unlock();
 668                         goto end_free_entries;
 669                 }
 670                 entries[i] = nullptr;
 671         }
 672         tracker->count.unsuspendable += fd_count;
 673         rcu_read_unlock();
 674         ret = user_ret;
 675 end_unlock:
 676         pthread_mutex_unlock(&tracker->lock);
 677 end:
 678         free(entries);
 679         return ret;
 680 end_free_entries:
 681         for (i = 0; i < fd_count; i++) {
 682                 unsuspendable_fd_destroy(entries[i]);
 683         }
 684         goto end_unlock;
 685 }
 686
 687 int fd_tracker_close_unsuspendable_fd(struct fd_tracker *tracker,
 688                                       int *fds_in,
 689                                       unsigned int fd_count,
 690                                       fd_close_cb close,
 691                                       void *user_data)
 692 {
 693         int i, ret, user_ret;
 694         int *fds = nullptr;
 695
 696         /*
 697          * Maintain a local copy of fds_in as the user's callback may modify its
 698          * contents (e.g. setting the fd(s) to -1 after close).
 699          */
 700         fds = malloc<int>(sizeof(*fds) * fd_count);
 701         if (!fds) {
 702                 ret = -1;
 703                 goto end;
 704         }
 705         memcpy(fds, fds_in, sizeof(*fds) * fd_count);
 706
 707         pthread_mutex_lock(&tracker->lock);
 708         rcu_read_lock();
 709
 710         /* Let the user close the file descriptors. */
 711         user_ret = close(user_data, fds_in);
 712         if (user_ret) {
 713                 ret = user_ret;
 714                 goto end_unlock;
 715         }
 716
 717         /* Untrack the fds that were just closed by the user's callback. */
 718         for (i = 0; i < fd_count; i++) {
 719                 struct cds_lfht_node *node;
 720                 struct cds_lfht_iter iter;
 721                 struct unsuspendable_fd *entry;
 722
 723                 cds_lfht_lookup(tracker->unsuspendable_fds,
 724                                 hash_key_ulong((void *) (unsigned long) fds[i], seed.value),
 725                                 match_fd,
 726                                 (void *) (unsigned long) fds[i],
 727                                 &iter);
 728                 node = cds_lfht_iter_get_node(&iter);
 729                 if (!node) {
 730                         /* Unknown file descriptor. */
 731                         WARN("Untracked file descriptor %d passed to fd_tracker_close_unsuspendable_fd()",
 732                              fds[i]);
 733                         ret = -EINVAL;
 734                         goto end_unlock;
 735                 }
 736                 entry = lttng::utils::container_of(node, &unsuspendable_fd::tracker_node);
 737
 738                 cds_lfht_del(tracker->unsuspendable_fds, node);
 739                 unsuspendable_fd_destroy(entry);
 740                 fds[i] = -1;
 741         }
 742
 743         tracker->count.unsuspendable -= fd_count;
 744         ret = 0;
 745 end_unlock:
 746         rcu_read_unlock();
 747         pthread_mutex_unlock(&tracker->lock);
 748         free(fds);
 749 end:
 750         return ret;
 751 }
 752
 753 /* Caller must have taken the tracker's and handle's locks. */
 754 static void fd_tracker_track(struct fd_tracker *tracker, struct fs_handle_tracked *handle)
 755 {
 756         if (handle->fd >= 0) {
 757                 tracker->count.suspendable.active++;
 758                 cds_list_add_tail(&handle->handles_list_node, &tracker->active_handles);
 759         } else {
 760                 tracker->count.suspendable.suspended++;
 761                 cds_list_add_tail(&handle->handles_list_node, &tracker->suspended_handles);
 762         }
 763 }
 764
 765 /* Caller must have taken the tracker's and handle's locks. */
 766 static void fd_tracker_untrack(struct fd_tracker *tracker, struct fs_handle_tracked *handle)
 767 {
 768         if (handle->fd >= 0) {
 769                 tracker->count.suspendable.active--;
 770         } else {
 771                 tracker->count.suspendable.suspended--;
 772         }
 773         cds_list_del(&handle->handles_list_node);
 774 }
 775
 776 /* Caller must have taken the tracker's and handle's locks. */
 777 static int fd_tracker_restore_handle(struct fd_tracker *tracker, struct fs_handle_tracked *handle)
 778 {
 779         int ret;
 780
 781         fd_tracker_untrack(tracker, handle);
 782         if (ACTIVE_COUNT(tracker) >= tracker->capacity) {
 783                 ret = fd_tracker_suspend_handles(tracker, 1);
 784                 if (ret) {
 785                         goto end;
 786                 }
 787         }
 788         ret = fs_handle_tracked_restore(handle);
 789 end:
 790         fd_tracker_track(tracker, handle);
 791         return ret ? ret : handle->fd;
 792 }
 793
 794 static int fs_handle_tracked_get_fd(struct fs_handle *_handle)
 795 {
 796         int ret;
 797         struct fs_handle_tracked *handle =
 798                 lttng::utils::container_of(_handle, &fs_handle_tracked::parent);
 799
 800         /*
 801          * TODO This should be optimized as it is a fairly hot path.
 802          * The fd-tracker's lock should only be taken when a fs_handle is
 803          * restored (slow path). On the fast path (fs_handle is active),
 804          * the only effect on the fd_tracker is marking the handle as the
 805          * most recently used. Currently, it is done by a call to the
 806          * track/untrack helpers, but it should be done atomically.
 807          *
 808          * Note that the lock's nesting order must still be respected here.
 809          * The handle's lock nests inside the tracker's lock.
 810          */
 811         pthread_mutex_lock(&handle->tracker->lock);
 812         pthread_mutex_lock(&handle->lock);
 813         LTTNG_ASSERT(!handle->in_use);
 814
 815         handle->tracker->stats.uses++;
 816         if (handle->fd >= 0) {
 817                 ret = handle->fd;
 818                 /* Mark as most recently used. */
 819                 fd_tracker_untrack(handle->tracker, handle);
 820                 fd_tracker_track(handle->tracker, handle);
 821         } else {
 822                 handle->tracker->stats.misses++;
 823                 ret = fd_tracker_restore_handle(handle->tracker, handle);
 824                 if (ret < 0) {
 825                         handle->tracker->stats.errors++;
 826                         goto end;
 827                 }
 828         }
 829         handle->in_use = true;
 830 end:
 831         pthread_mutex_unlock(&handle->lock);
 832         pthread_mutex_unlock(&handle->tracker->lock);
 833         return ret;
 834 }
 835
 836 static void fs_handle_tracked_put_fd(struct fs_handle *_handle)
 837 {
 838         struct fs_handle_tracked *handle =
 839                 lttng::utils::container_of(_handle, &fs_handle_tracked::parent);
 840
 841         pthread_mutex_lock(&handle->lock);
 842         handle->in_use = false;
 843         pthread_mutex_unlock(&handle->lock);
 844 }
 845
 846 static int fs_handle_tracked_unlink(struct fs_handle *_handle)
 847 {
 848         int ret;
 849         struct fs_handle_tracked *handle =
 850                 lttng::utils::container_of(_handle, &fs_handle_tracked::parent);
 851
 852         pthread_mutex_lock(&handle->tracker->lock);
 853         pthread_mutex_lock(&handle->lock);
 854         ret = lttng_inode_unlink(handle->inode);
 855         pthread_mutex_unlock(&handle->lock);
 856         pthread_mutex_unlock(&handle->tracker->lock);
 857         return ret;
 858 }
 859
 860 static int fs_handle_tracked_close(struct fs_handle *_handle)
 861 {
 862         int ret = 0;
 863         const char *path = nullptr;
 864         struct fs_handle_tracked *handle =
 865                 lttng::utils::container_of(_handle, &fs_handle_tracked::parent);
 866         struct lttng_directory_handle *inode_directory_handle = nullptr;
 867
 868         if (!handle) {
 869                 ret = -EINVAL;
 870                 goto end;
 871         }
 872
 873         pthread_mutex_lock(&handle->tracker->lock);
 874         pthread_mutex_lock(&handle->lock);
 875         if (handle->inode) {
 876                 lttng_inode_borrow_location(handle->inode, nullptr, &path);
 877                 /*
 878                  * Here a reference to the inode's directory handle is acquired
 879                  * to prevent the last reference to it from being released while
 880                  * the tracker's lock is taken.
 881                  *
 882                  * If this wasn't done, the directory handle could attempt to
 883                  * close its underlying directory file descriptor, which would
 884                  * attempt to lock the tracker's lock, resulting in a deadlock.
 885                  *
 886                  * Since a new reference to the directory handle is taken within
 887                  * the scope of this function, it is not possible for the last
 888                  * reference to the inode's location directory handle to be
 889                  * released during the call to lttng_inode_put().
 890                  *
 891                  * We wait until the tracker's lock is released to release the
 892                  * reference. Hence, the call to the tracker is delayed just
 893                  * enough to not attempt to recursively acquire the tracker's
 894                  * lock twice.
 895                  */
 896                 inode_directory_handle = lttng_inode_get_location_directory_handle(handle->inode);
 897         }
 898         fd_tracker_untrack(handle->tracker, handle);
 899         if (handle->fd >= 0) {
 900                 /*
 901                  * The return value of close() is not propagated as there
 902                  * isn't much the user can do about it.
 903                  */
 904                 if (close(handle->fd)) {
 905                         PERROR("Failed to close the file descriptor (%d) of fs handle to %s, close() returned",
 906                                handle->fd,
 907                                path ? path : "Unknown");
 908                 }
 909                 handle->fd = -1;
 910         }
 911         if (handle->inode) {
 912                 lttng_inode_put(handle->inode);
 913         }
 914         pthread_mutex_unlock(&handle->lock);
 915         pthread_mutex_destroy(&handle->lock);
 916         pthread_mutex_unlock(&handle->tracker->lock);
 917         free(handle);
 918         lttng_directory_handle_put(inode_directory_handle);
 919 end:
 920         return ret;
 921 }