2 * Copyright (C) 2013 David Goulet <dgoulet@efficios.com>
4 * SPDX-License-Identifier: GPL-2.0-only
9 #include "buffer-registry.hpp"
10 #include "fd-limit.hpp"
11 #include "lttng-ust-ctl.hpp"
12 #include "lttng-ust-error.hpp"
13 #include "ust-consumer.hpp"
16 #include <common/common.hpp>
17 #include <common/hashtable/utils.hpp>
18 #include <common/urcu.hpp>
23 * Set in main.c during initialization process of the daemon. This contains
24 * buffer_reg_uid object which are global registry for per UID buffer. Object
25 * are indexed by session id and matched by the triplet
26 * <session_id/bits_per_long/uid>.
28 static struct lttng_ht
*buffer_registry_uid
;
31 * Initialized at the daemon start. This contains buffer_reg_pid object and
32 * indexed by session id.
34 static struct lttng_ht
*buffer_registry_pid
;
37 * Match function for the per UID registry hash table. It matches a registry
38 * uid object with the triplet <session_id/abi/uid>.
40 static int ht_match_reg_uid(struct cds_lfht_node
*node
, const void *_key
)
45 auto *reg
= lttng_ht_node_container_of(node
, &buffer_reg_uid::node
);
46 const auto *key
= (buffer_reg_uid
*) _key
;
48 if (key
->session_id
!= reg
->session_id
|| key
->bits_per_long
!= reg
->bits_per_long
||
49 key
->uid
!= reg
->uid
) {
60 * Hash function for the per UID registry hash table. This XOR the triplet
63 static unsigned long ht_hash_reg_uid(const void *_key
, unsigned long seed
)
66 const struct buffer_reg_uid
*key
= (buffer_reg_uid
*) _key
;
70 xored_key
= (uint64_t) (key
->session_id
^ key
->bits_per_long
^ key
->uid
);
71 return hash_key_u64(&xored_key
, seed
);
75 * Initialize global buffer per UID registry. Should only be called ONCE!.
77 void buffer_reg_init_uid_registry()
79 /* Should be called once. */
80 LTTNG_ASSERT(!buffer_registry_uid
);
81 buffer_registry_uid
= lttng_ht_new(0, LTTNG_HT_TYPE_U64
);
82 LTTNG_ASSERT(buffer_registry_uid
);
83 buffer_registry_uid
->match_fct
= ht_match_reg_uid
;
84 buffer_registry_uid
->hash_fct
= ht_hash_reg_uid
;
86 DBG3("Global buffer per UID registry initialized");
90 * Allocate and initialize object. Set regp with the object pointer.
92 * Return 0 on success else a negative value and regp is untouched.
94 int buffer_reg_uid_create(uint64_t session_id
,
95 uint32_t bits_per_long
,
97 enum lttng_domain_type domain
,
98 struct buffer_reg_uid
**regp
,
99 const char *root_shm_path
,
100 const char *shm_path
)
103 struct buffer_reg_uid
*reg
= nullptr;
107 reg
= zmalloc
<buffer_reg_uid
>();
109 PERROR("zmalloc buffer registry uid");
114 reg
->registry
= zmalloc
<buffer_reg_session
>();
115 if (!reg
->registry
) {
116 PERROR("zmalloc buffer registry uid session");
121 reg
->session_id
= session_id
;
122 reg
->bits_per_long
= bits_per_long
;
124 reg
->domain
= domain
;
126 strncpy(reg
->root_shm_path
, root_shm_path
, sizeof(reg
->root_shm_path
));
127 reg
->root_shm_path
[sizeof(reg
->root_shm_path
) - 1] = '\0';
128 strncpy(reg
->shm_path
, shm_path
, sizeof(reg
->shm_path
));
129 reg
->shm_path
[sizeof(reg
->shm_path
) - 1] = '\0';
130 DBG3("shm path '%s' is assigned to uid buffer registry for session id %" PRIu64
,
134 reg
->registry
->channels
= lttng_ht_new(0, LTTNG_HT_TYPE_U64
);
135 if (!reg
->registry
->channels
) {
140 cds_lfht_node_init(®
->node
.node
);
143 DBG3("Buffer registry per UID created id: %" PRIu64
", ABI: %u, uid: %d, domain: %d",
159 * Add a buffer registry per UID object to the global registry.
161 void buffer_reg_uid_add(struct buffer_reg_uid
*reg
)
163 struct cds_lfht_node
*nodep
;
164 struct lttng_ht
*ht
= buffer_registry_uid
;
168 DBG3("Buffer registry per UID adding to global registry with id: %" PRIu64
,
171 const lttng::urcu::read_lock_guard read_lock
;
172 nodep
= cds_lfht_add_unique(
173 ht
->ht
, ht
->hash_fct(reg
, lttng_ht_seed
), ht
->match_fct
, reg
, ®
->node
.node
);
174 LTTNG_ASSERT(nodep
== ®
->node
.node
);
178 * Find a buffer registry per UID object with given params. RCU read side lock
179 * MUST be acquired before calling this and hold on to protect the object.
181 * Return the object pointer or NULL on error.
183 struct buffer_reg_uid
*buffer_reg_uid_find(uint64_t session_id
, uint32_t bits_per_long
, uid_t uid
)
185 struct lttng_ht_node_u64
*node
;
186 struct lttng_ht_iter iter
;
187 struct buffer_reg_uid
*reg
= nullptr, key
;
188 struct lttng_ht
*ht
= buffer_registry_uid
;
190 ASSERT_RCU_READ_LOCKED();
192 /* Setup key we are looking for. */
193 key
.session_id
= session_id
;
194 key
.bits_per_long
= bits_per_long
;
197 DBG3("Buffer registry per UID find id: %" PRIu64
", ABI: %u, uid: %d",
202 /* Custom lookup function since it's a different key. */
203 cds_lfht_lookup(ht
->ht
, ht
->hash_fct(&key
, lttng_ht_seed
), ht
->match_fct
, &key
, &iter
.iter
);
204 node
= lttng_ht_iter_get_node
<lttng_ht_node_u64
>(&iter
);
208 reg
= lttng::utils::container_of(node
, &buffer_reg_uid::node
);
215 * Initialize global buffer per PID registry. Should only be called ONCE!.
217 void buffer_reg_init_pid_registry()
219 /* Should be called once. */
220 LTTNG_ASSERT(!buffer_registry_pid
);
221 buffer_registry_pid
= lttng_ht_new(0, LTTNG_HT_TYPE_U64
);
222 LTTNG_ASSERT(buffer_registry_pid
);
224 DBG3("Global buffer per PID registry initialized");
228 * Allocate and initialize object. Set regp with the object pointer.
230 * Return 0 on success else a negative value and regp is untouched.
232 int buffer_reg_pid_create(uint64_t session_id
,
233 struct buffer_reg_pid
**regp
,
234 const char *root_shm_path
,
235 const char *shm_path
)
238 struct buffer_reg_pid
*reg
= nullptr;
242 reg
= zmalloc
<buffer_reg_pid
>();
244 PERROR("zmalloc buffer registry pid");
249 reg
->registry
= zmalloc
<buffer_reg_session
>();
250 if (!reg
->registry
) {
251 PERROR("zmalloc buffer registry pid session");
256 /* A cast is done here so we can use the session ID as a u64 ht node. */
257 reg
->session_id
= session_id
;
259 strncpy(reg
->root_shm_path
, root_shm_path
, sizeof(reg
->root_shm_path
));
260 reg
->root_shm_path
[sizeof(reg
->root_shm_path
) - 1] = '\0';
261 strncpy(reg
->shm_path
, shm_path
, sizeof(reg
->shm_path
));
262 reg
->shm_path
[sizeof(reg
->shm_path
) - 1] = '\0';
263 DBG3("shm path '%s' is assigned to pid buffer registry for session id %" PRIu64
,
267 reg
->registry
->channels
= lttng_ht_new(0, LTTNG_HT_TYPE_U64
);
268 if (!reg
->registry
->channels
) {
273 lttng_ht_node_init_u64(®
->node
, reg
->session_id
);
276 DBG3("Buffer registry per PID created with session id: %" PRIu64
, session_id
);
288 * Add a buffer registry per PID object to the global registry.
290 void buffer_reg_pid_add(struct buffer_reg_pid
*reg
)
294 DBG3("Buffer registry per PID adding to global registry with id: %" PRIu64
,
297 const lttng::urcu::read_lock_guard read_lock
;
298 lttng_ht_add_unique_u64(buffer_registry_pid
, ®
->node
);
302 * Find a buffer registry per PID object with given params. RCU read side lock
303 * MUST be acquired before calling this and hold on to protect the object.
305 * Return the object pointer or NULL on error.
307 struct buffer_reg_pid
*buffer_reg_pid_find(uint64_t session_id
)
309 struct lttng_ht_node_u64
*node
;
310 struct lttng_ht_iter iter
;
311 struct buffer_reg_pid
*reg
= nullptr;
312 struct lttng_ht
*ht
= buffer_registry_pid
;
314 DBG3("Buffer registry per PID find id: %" PRIu64
, session_id
);
316 lttng_ht_lookup(ht
, &session_id
, &iter
);
317 node
= lttng_ht_iter_get_node
<lttng_ht_node_u64
>(&iter
);
321 reg
= lttng::utils::container_of(node
, &buffer_reg_pid::node
);
328 * Find the consumer channel key from a UST session per-uid channel key.
330 * Return the matching key or -1 if not found.
332 int buffer_reg_uid_consumer_channel_key(struct cds_list_head
*buffer_reg_uid_list
,
334 uint64_t *consumer_chan_key
)
339 * For the per-uid registry, we have to iterate since we don't have the
340 * uid and bitness key.
343 lttng::urcu::list_iteration_adapter
<buffer_reg_uid
, &buffer_reg_uid::lnode
>(
344 *buffer_reg_uid_list
)) {
345 auto *session_reg
= uid_reg
->registry
;
346 for (auto *reg_chan
:
347 lttng::urcu::lfht_iteration_adapter
<buffer_reg_channel
,
348 decltype(buffer_reg_channel::node
),
349 &buffer_reg_channel::node
>(
350 *session_reg
->channels
->ht
)) {
351 if (reg_chan
->key
== chan_key
) {
352 *consumer_chan_key
= reg_chan
->consumer_key
;
363 * Allocate and initialize a buffer registry channel with the given key. Set
364 * regp with the object pointer.
366 * Return 0 on success or else a negative value keeping regp untouched.
368 int buffer_reg_channel_create(uint64_t key
, struct buffer_reg_channel
**regp
)
370 struct buffer_reg_channel
*reg
;
374 DBG3("Buffer registry channel create with key: %" PRIu64
, key
);
376 reg
= zmalloc
<buffer_reg_channel
>();
378 PERROR("zmalloc buffer registry channel");
383 CDS_INIT_LIST_HEAD(®
->streams
);
384 pthread_mutex_init(®
->stream_list_lock
, nullptr);
386 lttng_ht_node_init_u64(®
->node
, key
);
393 * Allocate and initialize a buffer registry stream. Set regp with the object
396 * Return 0 on success or else a negative value keeping regp untouched.
398 int buffer_reg_stream_create(struct buffer_reg_stream
**regp
)
400 struct buffer_reg_stream
*reg
;
404 DBG3("Buffer registry creating stream");
406 reg
= zmalloc
<buffer_reg_stream
>();
408 PERROR("zmalloc buffer registry stream");
418 * Add stream to the list in the channel.
420 void buffer_reg_stream_add(struct buffer_reg_stream
*stream
, struct buffer_reg_channel
*channel
)
422 LTTNG_ASSERT(stream
);
423 LTTNG_ASSERT(channel
);
425 pthread_mutex_lock(&channel
->stream_list_lock
);
426 cds_list_add_tail(&stream
->lnode
, &channel
->streams
);
427 channel
->stream_count
++;
428 pthread_mutex_unlock(&channel
->stream_list_lock
);
432 * Add a buffer registry channel object to the given session.
434 void buffer_reg_channel_add(struct buffer_reg_session
*session
, struct buffer_reg_channel
*channel
)
436 LTTNG_ASSERT(session
);
437 LTTNG_ASSERT(channel
);
439 const lttng::urcu::read_lock_guard read_lock
;
440 lttng_ht_add_unique_u64(session
->channels
, &channel
->node
);
444 * Find a buffer registry channel object with the given key. RCU read side lock
445 * MUST be acquired and hold on until the object reference is not needed
448 * Return the object pointer or NULL on error.
450 struct buffer_reg_channel
*buffer_reg_channel_find(uint64_t key
, struct buffer_reg_uid
*reg
)
452 struct lttng_ht_node_u64
*node
;
453 struct lttng_ht_iter iter
;
454 struct buffer_reg_channel
*chan
= nullptr;
459 switch (reg
->domain
) {
460 case LTTNG_DOMAIN_UST
:
461 ht
= reg
->registry
->channels
;
468 lttng_ht_lookup(ht
, &key
, &iter
);
469 node
= lttng_ht_iter_get_node
<lttng_ht_node_u64
>(&iter
);
473 chan
= lttng::utils::container_of(node
, &buffer_reg_channel::node
);
480 * Destroy a buffer registry stream with the given domain.
482 void buffer_reg_stream_destroy(struct buffer_reg_stream
*regp
, enum lttng_domain_type domain
)
488 DBG3("Buffer registry stream destroy with handle %d", regp
->obj
.ust
->handle
);
491 case LTTNG_DOMAIN_UST
:
495 ret
= ust_app_release_object(nullptr, regp
->obj
.ust
);
496 if (ret
< 0 && ret
!= -EPIPE
&& ret
!= -LTTNG_UST_ERR_EXITING
) {
497 ERR("Buffer reg stream release obj handle %d failed with ret %d",
498 regp
->obj
.ust
->handle
,
502 lttng_fd_put(LTTNG_FD_APPS
, 2);
514 * Remove buffer registry channel object from the session hash table. RCU read
515 * side lock MUST be acquired before calling this.
517 void buffer_reg_channel_remove(struct buffer_reg_session
*session
, struct buffer_reg_channel
*regp
)
520 struct lttng_ht_iter iter
;
522 LTTNG_ASSERT(session
);
525 iter
.iter
.node
= ®p
->node
.node
;
526 ret
= lttng_ht_del(session
->channels
, &iter
);
531 * Destroy a buffer registry channel with the given domain.
533 void buffer_reg_channel_destroy(struct buffer_reg_channel
*regp
, enum lttng_domain_type domain
)
539 DBG3("Buffer registry channel destroy with key %" PRIu32
, regp
->key
);
542 case LTTNG_DOMAIN_UST
:
547 for (auto reg_stream
:
548 lttng::urcu::list_iteration_adapter
<buffer_reg_stream
,
549 &buffer_reg_stream::lnode
>(
551 cds_list_del(®_stream
->lnode
);
552 regp
->stream_count
--;
553 buffer_reg_stream_destroy(reg_stream
, domain
);
557 ret
= ust_app_release_object(nullptr, regp
->obj
.ust
);
558 if (ret
< 0 && ret
!= -EPIPE
&& ret
!= -LTTNG_UST_ERR_EXITING
) {
559 ERR("Buffer reg channel release obj handle %d failed with ret %d",
560 regp
->obj
.ust
->handle
,
565 lttng_fd_put(LTTNG_FD_APPS
, 1);
577 * Destroy a buffer registry session with the given domain.
579 static void buffer_reg_session_destroy(struct buffer_reg_session
*regp
,
580 enum lttng_domain_type domain
)
582 DBG3("Buffer registry session destroy");
584 /* Destroy all channels. */
585 for (auto *reg_chan
:
586 lttng::urcu::lfht_iteration_adapter
<buffer_reg_channel
,
587 decltype(buffer_reg_channel::node
),
588 &buffer_reg_channel::node
>(*regp
->channels
->ht
)) {
589 const auto ret
= cds_lfht_del(regp
->channels
->ht
, ®_chan
->node
.node
);
591 buffer_reg_channel_destroy(reg_chan
, domain
);
594 lttng_ht_destroy(regp
->channels
);
597 case LTTNG_DOMAIN_UST
:
598 ust_registry_session_destroy(regp
->reg
.ust
);
609 * Remove buffer registry UID object from the global hash table.
611 void buffer_reg_uid_remove(struct buffer_reg_uid
*regp
)
614 struct lttng_ht_iter iter
;
618 const lttng::urcu::read_lock_guard read_lock
;
619 iter
.iter
.node
= ®p
->node
.node
;
620 ret
= lttng_ht_del(buffer_registry_uid
, &iter
);
624 static void rcu_free_buffer_reg_uid(struct rcu_head
*head
)
626 struct lttng_ht_node_u64
*node
= lttng::utils::container_of(head
, <tng_ht_node_u64::head
);
627 struct buffer_reg_uid
*reg
= lttng::utils::container_of(node
, &buffer_reg_uid::node
);
629 buffer_reg_session_destroy(reg
->registry
, reg
->domain
);
633 static void rcu_free_buffer_reg_pid(struct rcu_head
*head
)
635 struct lttng_ht_node_u64
*node
= lttng::utils::container_of(head
, <tng_ht_node_u64::head
);
636 struct buffer_reg_pid
*reg
= lttng::utils::container_of(node
, &buffer_reg_pid::node
);
638 buffer_reg_session_destroy(reg
->registry
, LTTNG_DOMAIN_UST
);
643 * Destroy buffer registry per UID. The given pointer is NOT removed from any
644 * list or hash table. Use buffer_reg_pid_remove() before calling this function
645 * for the case that the object is in the global hash table.
647 void buffer_reg_uid_destroy(struct buffer_reg_uid
*regp
, struct consumer_output
*consumer
)
649 struct consumer_socket
*socket
;
655 DBG3("Buffer registry per UID destroy with id: %" PRIu64
", ABI: %u, uid: %d",
665 const lttng::urcu::read_lock_guard read_lock
;
666 /* Get the right socket from the consumer object. */
667 socket
= consumer_find_socket_by_bitness(regp
->bits_per_long
, consumer
);
672 switch (regp
->domain
) {
673 case LTTNG_DOMAIN_UST
:
674 if (regp
->registry
->reg
.ust
->_metadata_key
) {
675 /* Return value does not matter. This call will print errors. */
676 (void) consumer_close_metadata(
677 socket
, regp
->registry
->reg
.ust
->_metadata_key
);
687 call_rcu(®p
->node
.head
, rcu_free_buffer_reg_uid
);
691 * Remove buffer registry UID object from the global hash table. RCU read side
692 * lock MUST be acquired before calling this.
694 void buffer_reg_pid_remove(struct buffer_reg_pid
*regp
)
697 struct lttng_ht_iter iter
;
701 iter
.iter
.node
= ®p
->node
.node
;
702 ret
= lttng_ht_del(buffer_registry_pid
, &iter
);
707 * Destroy buffer registry per PID. The pointer is NOT removed from the global
708 * hash table. Call buffer_reg_pid_remove() before that if the object was
709 * previously added to the global hash table.
711 void buffer_reg_pid_destroy(struct buffer_reg_pid
*regp
)
717 DBG3("Buffer registry per PID destroy with id: %" PRIu64
, regp
->session_id
);
719 /* This registry is only used by UST. */
720 call_rcu(®p
->node
.head
, rcu_free_buffer_reg_pid
);
724 * Destroy per PID and UID registry hash table.
726 void buffer_reg_destroy_registries()
728 DBG3("Buffer registry destroy all registry");
729 lttng_ht_destroy(buffer_registry_uid
);
730 lttng_ht_destroy(buffer_registry_pid
);