/*
- * Copyright (C) 2017 - Julien Desfossez <jdesfossez@efficios.com>
+ * Copyright (C) 2017 Julien Desfossez <jdesfossez@efficios.com>
+ * Copyright (C) 2018 Jérémie Galarneau <jeremie.galarneau@efficios.com>
*
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License, version 2 only, as
- * published by the Free Software Foundation.
+ * SPDX-License-Identifier: GPL-2.0-only
*
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 51
- * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define _LGPL_SOURCE
#include <common/align.h>
#include <common/time.h>
#include <common/hashtable/utils.h>
-#include <sys/eventfd.h>
#include <sys/stat.h>
#include <time.h>
#include <signal.h>
#include <common/kernel-ctl/kernel-ctl.h>
#include <lttng/notification/channel-internal.h>
#include <lttng/rotate-internal.h>
+#include <lttng/location-internal.h>
+#include <lttng/condition/condition-internal.h>
#include "rotation-thread.h"
#include "lttng-sessiond.h"
#include "rotate.h"
#include "cmd.h"
#include "session.h"
-#include "sessiond-timer.h"
+#include "timer.h"
+#include "notification-thread-commands.h"
+#include "utils.h"
+#include "thread.h"
#include <urcu.h>
#include <urcu/list.h>
-#include <urcu/rculfhash.h>
-
-/*
- * Store a struct rotation_channel_info for each channel that is currently
- * being rotated by the consumer.
- */
-struct cds_lfht *channel_pending_rotate_ht;
struct lttng_notification_channel *rotate_notification_channel = NULL;
-struct rotation_thread_state {
+struct rotation_thread {
struct lttng_poll_event events;
};
+struct rotation_thread_job {
+ enum rotation_thread_job_type type;
+ struct ltt_session *session;
+ /* List member in struct rotation_thread_timer_queue. */
+ struct cds_list_head head;
+};
+
+/*
+ * The timer thread enqueues jobs and wakes up the rotation thread.
+ * When the rotation thread wakes up, it empties the queue.
+ */
+struct rotation_thread_timer_queue {
+ struct lttng_pipe *event_pipe;
+ struct cds_list_head list;
+ pthread_mutex_t lock;
+};
+
+struct rotation_thread_handle {
+ struct rotation_thread_timer_queue *rotation_timer_queue;
+ /* Access to the notification thread cmd_queue */
+ struct notification_thread_handle *notification_thread_handle;
+ /* Thread-specific quit pipe. */
+ struct lttng_pipe *quit_pipe;
+};
+
static
-void channel_rotation_info_destroy(struct rotation_channel_info *channel_info)
+const char *get_job_type_str(enum rotation_thread_job_type job_type)
{
- assert(channel_info);
- free(channel_info);
+ switch (job_type) {
+ case ROTATION_THREAD_JOB_TYPE_CHECK_PENDING_ROTATION:
+ return "CHECK_PENDING_ROTATION";
+ case ROTATION_THREAD_JOB_TYPE_SCHEDULED_ROTATION:
+ return "SCHEDULED_ROTATION";
+ default:
+ abort();
+ }
}
-static
-int match_channel_info(struct cds_lfht_node *node, const void *key)
+struct rotation_thread_timer_queue *rotation_thread_timer_queue_create(void)
{
- struct rotation_channel_key *channel_key = (struct rotation_channel_key *) key;
- struct rotation_channel_info *channel_info;
+ struct rotation_thread_timer_queue *queue = NULL;
- channel_info = caa_container_of(node, struct rotation_channel_info,
- rotate_channels_ht_node);
+ queue = zmalloc(sizeof(*queue));
+ if (!queue) {
+ PERROR("Failed to allocate timer rotate queue");
+ goto end;
+ }
- return !!((channel_key->key == channel_info->channel_key.key) &&
- (channel_key->domain == channel_info->channel_key.domain));
+ queue->event_pipe = lttng_pipe_open(FD_CLOEXEC | O_NONBLOCK);
+ CDS_INIT_LIST_HEAD(&queue->list);
+ pthread_mutex_init(&queue->lock, NULL);
+end:
+ return queue;
}
-static
-struct rotation_channel_info *lookup_channel_pending(uint64_t key,
- enum lttng_domain_type domain)
+void rotation_thread_timer_queue_destroy(
+ struct rotation_thread_timer_queue *queue)
{
- struct cds_lfht_iter iter;
- struct cds_lfht_node *node;
- struct rotation_channel_info *channel_info = NULL;
- struct rotation_channel_key channel_key = { .key = key,
- .domain = domain };
-
- cds_lfht_lookup(channel_pending_rotate_ht,
- hash_channel_key(&channel_key),
- match_channel_info,
- &channel_key, &iter);
- node = cds_lfht_iter_get_node(&iter);
- if (!node) {
- goto end;
+ if (!queue) {
+ return;
}
- channel_info = caa_container_of(node, struct rotation_channel_info,
- rotate_channels_ht_node);
- cds_lfht_del(channel_pending_rotate_ht, node);
-end:
- return channel_info;
+ lttng_pipe_destroy(queue->event_pipe);
+
+ pthread_mutex_lock(&queue->lock);
+ assert(cds_list_empty(&queue->list));
+ pthread_mutex_unlock(&queue->lock);
+ pthread_mutex_destroy(&queue->lock);
+ free(queue);
}
/*
void rotation_thread_handle_destroy(
struct rotation_thread_handle *handle)
{
- int ret;
-
- if (!handle) {
- goto end;
- }
-
- if (handle->ust32_consumer >= 0) {
- ret = close(handle->ust32_consumer);
- if (ret) {
- PERROR("close 32-bit consumer channel rotation pipe");
- }
- }
- if (handle->ust64_consumer >= 0) {
- ret = close(handle->ust64_consumer);
- if (ret) {
- PERROR("close 64-bit consumer channel rotation pipe");
- }
- }
- if (handle->kernel_consumer >= 0) {
- ret = close(handle->kernel_consumer);
- if (ret) {
- PERROR("close kernel consumer channel rotation pipe");
- }
- }
-
-end:
+ lttng_pipe_destroy(handle->quit_pipe);
free(handle);
}
struct rotation_thread_handle *rotation_thread_handle_create(
- struct lttng_pipe *ust32_channel_rotate_pipe,
- struct lttng_pipe *ust64_channel_rotate_pipe,
- struct lttng_pipe *kernel_channel_rotate_pipe,
- int thread_quit_pipe,
struct rotation_thread_timer_queue *rotation_timer_queue,
- struct notification_thread_handle *notification_thread_handle,
- sem_t *notification_thread_ready)
+ struct notification_thread_handle *notification_thread_handle)
{
struct rotation_thread_handle *handle;
goto end;
}
- if (ust32_channel_rotate_pipe) {
- handle->ust32_consumer =
- lttng_pipe_release_readfd(
- ust32_channel_rotate_pipe);
- if (handle->ust32_consumer < 0) {
- goto error;
- }
- } else {
- handle->ust32_consumer = -1;
- }
- if (ust64_channel_rotate_pipe) {
- handle->ust64_consumer =
- lttng_pipe_release_readfd(
- ust64_channel_rotate_pipe);
- if (handle->ust64_consumer < 0) {
- goto error;
- }
- } else {
- handle->ust64_consumer = -1;
- }
- if (kernel_channel_rotate_pipe) {
- handle->kernel_consumer =
- lttng_pipe_release_readfd(
- kernel_channel_rotate_pipe);
- if (handle->kernel_consumer < 0) {
- goto error;
- }
- } else {
- handle->kernel_consumer = -1;
- }
- handle->thread_quit_pipe = thread_quit_pipe;
handle->rotation_timer_queue = rotation_timer_queue;
handle->notification_thread_handle = notification_thread_handle;
- handle->notification_thread_ready = notification_thread_ready;
+ handle->quit_pipe = lttng_pipe_open(FD_CLOEXEC);
+ if (!handle->quit_pipe) {
+ goto error;
+ }
end:
return handle;
return NULL;
}
+/*
+ * Called with the rotation_thread_timer_queue lock held.
+ * Return true if the same timer job already exists in the queue, false if not.
+ */
+static
+bool timer_job_exists(const struct rotation_thread_timer_queue *queue,
+ enum rotation_thread_job_type job_type,
+ struct ltt_session *session)
+{
+ bool exists = false;
+ struct rotation_thread_job *job;
+
+ cds_list_for_each_entry(job, &queue->list, head) {
+ if (job->session == session && job->type == job_type) {
+ exists = true;
+ goto end;
+ }
+ }
+end:
+ return exists;
+}
+
+void rotation_thread_enqueue_job(struct rotation_thread_timer_queue *queue,
+ enum rotation_thread_job_type job_type,
+ struct ltt_session *session)
+{
+ int ret;
+ const char dummy = '!';
+ struct rotation_thread_job *job = NULL;
+ const char *job_type_str = get_job_type_str(job_type);
+
+ pthread_mutex_lock(&queue->lock);
+ if (timer_job_exists(queue, job_type, session)) {
+ /*
+ * This timer job is already pending, we don't need to add
+ * it.
+ */
+ goto end;
+ }
+
+ job = zmalloc(sizeof(struct rotation_thread_job));
+ if (!job) {
+ PERROR("Failed to allocate rotation thread job of type \"%s\" for session \"%s\"",
+ job_type_str, session->name);
+ goto end;
+ }
+ /* No reason for this to fail as the caller must hold a reference. */
+ (void) session_get(session);
+
+ job->session = session;
+ job->type = job_type;
+ cds_list_add_tail(&job->head, &queue->list);
+
+ ret = lttng_write(lttng_pipe_get_writefd(queue->event_pipe), &dummy,
+ sizeof(dummy));
+ if (ret < 0) {
+ /*
+ * We do not want to block in the timer handler, the job has
+ * been enqueued in the list, the wakeup pipe is probably full,
+ * the job will be processed when the rotation_thread catches
+ * up.
+ */
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ /*
+ * Not an error, but would be surprising and indicate
+ * that the rotation thread can't keep up with the
+ * current load.
+ */
+ DBG("Wake-up pipe of rotation thread job queue is full");
+ goto end;
+ }
+ PERROR("Failed to wake-up the rotation thread after pushing a job of type \"%s\" for session \"%s\"",
+ job_type_str, session->name);
+ goto end;
+ }
+
+end:
+ pthread_mutex_unlock(&queue->lock);
+}
+
static
int init_poll_set(struct lttng_poll_event *poll_set,
struct rotation_thread_handle *handle)
int ret;
/*
- * Create pollset with size 5:
- * - sessiond quit pipe
- * - sessiond timer pipe,
- * - consumerd (32-bit user space) channel rotate pipe,
- * - consumerd (64-bit user space) channel rotate pipe,
- * - consumerd (kernel) channel rotate pipe,
+ * Create pollset with size 3:
+ * - rotation thread quit pipe,
+ * - rotation thread timer queue pipe,
+ * - notification channel sock,
*/
ret = lttng_poll_create(poll_set, 5, LTTNG_CLOEXEC);
if (ret < 0) {
- goto end;
+ goto error;
}
- ret = lttng_poll_add(poll_set, handle->thread_quit_pipe,
+ ret = lttng_poll_add(poll_set,
+ lttng_pipe_get_readfd(handle->quit_pipe),
LPOLLIN | LPOLLERR);
if (ret < 0) {
- ERR("[rotation-thread] Failed to add thread_quit_pipe fd to pollset");
+ ERR("Failed to add quit pipe read fd to poll set");
goto error;
}
+
ret = lttng_poll_add(poll_set,
lttng_pipe_get_readfd(handle->rotation_timer_queue->event_pipe),
LPOLLIN | LPOLLERR);
if (ret < 0) {
- ERR("[rotation-thread] Failed to add rotate_pending fd to pollset");
- goto error;
- }
- ret = lttng_poll_add(poll_set, handle->ust32_consumer,
- LPOLLIN | LPOLLERR);
- if (ret < 0) {
- ERR("[rotation-thread] Failed to add ust-32 channel rotation pipe fd to pollset");
+ ERR("Failed to add rotate_pending fd to poll set");
goto error;
}
- ret = lttng_poll_add(poll_set, handle->ust64_consumer,
- LPOLLIN | LPOLLERR);
- if (ret < 0) {
- ERR("[rotation-thread] Failed to add ust-64 channel rotation pipe fd to pollset");
- goto error;
- }
- if (handle->kernel_consumer >= 0) {
- ret = lttng_poll_add(poll_set, handle->kernel_consumer,
- LPOLLIN | LPOLLERR);
- if (ret < 0) {
- ERR("[rotation-thread] Failed to add kernel channel rotation pipe fd to pollset");
- goto error;
- }
- }
-end:
return ret;
error:
lttng_poll_clean(poll_set);
}
static
-void fini_thread_state(struct rotation_thread_state *state)
+void fini_thread_state(struct rotation_thread *state)
{
lttng_poll_clean(&state->events);
- cds_lfht_destroy(channel_pending_rotate_ht, NULL);
if (rotate_notification_channel) {
lttng_notification_channel_destroy(rotate_notification_channel);
}
static
int init_thread_state(struct rotation_thread_handle *handle,
- struct rotation_thread_state *state)
+ struct rotation_thread *state)
{
int ret;
ret = init_poll_set(&state->events, handle);
if (ret) {
- ERR("[rotation-thread] Failed to initialize rotation thread poll set");
+ ERR("Failed to initialize rotation thread poll set");
goto end;
}
- channel_pending_rotate_ht = cds_lfht_new(DEFAULT_HT_SIZE,
- 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, NULL);
- if (!channel_pending_rotate_ht) {
- ERR("[rotation-thread] Failed to create channel pending rotation hash table");
- ret = -1;
- goto end;
- }
-
- /*
- * We wait until the notification thread is ready to create the
- * notification channel and add it to the poll_set.
- */
- sem_wait(handle->notification_thread_ready);
rotate_notification_channel = lttng_notification_channel_create(
lttng_session_daemon_notification_endpoint);
if (!rotate_notification_channel) {
- ERR("[rotation-thread] Could not create notification channel");
+ ERR("Could not create notification channel");
ret = -1;
goto end;
}
ret = lttng_poll_add(&state->events, rotate_notification_channel->socket,
LPOLLIN | LPOLLERR);
if (ret < 0) {
- ERR("[rotation-thread] Failed to add notification fd to pollset");
+ ERR("Failed to add notification fd to pollset");
goto end;
}
}
static
-int handle_channel_rotation_pipe(int fd, uint32_t revents,
- struct rotation_thread_handle *handle,
- struct rotation_thread_state *state)
+void check_session_rotation_pending_on_consumers(struct ltt_session *session,
+ bool *_rotation_completed)
{
int ret = 0;
- enum lttng_domain_type domain;
- struct rotation_channel_info *channel_info;
- struct ltt_session *session = NULL;
- uint64_t key;
-
- if (fd == handle->ust32_consumer ||
- fd == handle->ust64_consumer) {
- domain = LTTNG_DOMAIN_UST;
- } else if (fd == handle->kernel_consumer) {
- domain = LTTNG_DOMAIN_KERNEL;
- } else {
- ERR("[rotation-thread] Unknown channel rotation pipe fd %d",
- fd);
- abort();
- }
+ struct consumer_socket *socket;
+ struct cds_lfht_iter iter;
+ enum consumer_trace_chunk_exists_status exists_status;
+ uint64_t relayd_id;
+ bool chunk_exists_on_peer = false;
+ enum lttng_trace_chunk_status chunk_status;
- if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
- ret = lttng_poll_del(&state->events, fd);
+ assert(session->chunk_being_archived);
+
+ /*
+ * Check for a local pending rotation on all consumers (32-bit
+ * user space, 64-bit user space, and kernel).
+ */
+ rcu_read_lock();
+ if (!session->ust_session) {
+ goto skip_ust;
+ }
+ cds_lfht_for_each_entry(session->ust_session->consumer->socks->ht,
+ &iter, socket, node.node) {
+ relayd_id = session->ust_session->consumer->type == CONSUMER_DST_LOCAL ?
+ -1ULL :
+ session->ust_session->consumer->net_seq_index;
+
+ pthread_mutex_lock(socket->lock);
+ ret = consumer_trace_chunk_exists(socket,
+ relayd_id,
+ session->id, session->chunk_being_archived,
+ &exists_status);
if (ret) {
- ERR("[rotation-thread] Failed to remove consumer "
- "rotation pipe from poll set");
+ pthread_mutex_unlock(socket->lock);
+ ERR("Error occurred while checking rotation status on consumer daemon");
+ goto end;
}
- goto end;
- }
- do {
- ret = read(fd, &key, sizeof(key));
- } while (ret == -1 && errno == EINTR);
- if (ret != sizeof(key)) {
- ERR("[rotation-thread] Failed to read from pipe (fd = %i)",
- fd);
- ret = -1;
- goto end;
+ if (exists_status != CONSUMER_TRACE_CHUNK_EXISTS_STATUS_UNKNOWN_CHUNK) {
+ pthread_mutex_unlock(socket->lock);
+ chunk_exists_on_peer = true;
+ goto end;
+ }
+ pthread_mutex_unlock(socket->lock);
}
- DBG("[rotation-thread] Received notification for chan %" PRIu64
- ", domain %d", key, domain);
-
- channel_info = lookup_channel_pending(key, domain);
- if (!channel_info) {
- ERR("[rotation-thread] Failed to find channel_info (key = %"
- PRIu64 ")", key);
- ret = -1;
- goto end;
- }
- rcu_read_lock();
- session_lock_list();
- session = session_find_by_id(channel_info->session_id);
- if (!session) {
- /*
- * The session may have been destroyed before we had a chance to
- * perform this action, return gracefully.
- */
- DBG("[rotation-thread] Session %" PRIu64 " not found",
- channel_info->session_id);
- ret = 0;
- goto end_unlock_session_list;
+skip_ust:
+ if (!session->kernel_session) {
+ goto skip_kernel;
}
+ cds_lfht_for_each_entry(session->kernel_session->consumer->socks->ht,
+ &iter, socket, node.node) {
+ pthread_mutex_lock(socket->lock);
+ relayd_id = session->kernel_session->consumer->type == CONSUMER_DST_LOCAL ?
+ -1ULL :
+ session->kernel_session->consumer->net_seq_index;
- session_lock(session);
- if (--session->nr_chan_rotate_pending == 0) {
- time_t now = time(NULL);
-
- if (now == (time_t) -1) {
- session->rotation_state = LTTNG_ROTATION_STATE_ERROR;
- ret = LTTNG_ERR_UNK;
- goto end_unlock_session;
+ ret = consumer_trace_chunk_exists(socket,
+ relayd_id,
+ session->id, session->chunk_being_archived,
+ &exists_status);
+ if (ret) {
+ pthread_mutex_unlock(socket->lock);
+ ERR("Error occurred while checking rotation status on consumer daemon");
+ goto end;
}
- ret = rename_complete_chunk(session, now);
- if (ret < 0) {
- ERR("Failed to rename completed rotation chunk");
- goto end_unlock_session;
- }
- session->rotate_pending = false;
- session->last_chunk_start_ts = session->current_chunk_start_ts;
- if (session->rotate_pending_relay) {
- ret = sessiond_timer_rotate_pending_start(
- session,
- DEFAULT_ROTATE_PENDING_RELAY_TIMER);
- if (ret) {
- ERR("Failed to enable rotate pending timer");
- ret = -1;
- goto end_unlock_session;
- }
- } else {
- session->rotation_state = LTTNG_ROTATION_STATE_COMPLETED;
+ if (exists_status != CONSUMER_TRACE_CHUNK_EXISTS_STATUS_UNKNOWN_CHUNK) {
+ pthread_mutex_unlock(socket->lock);
+ chunk_exists_on_peer = true;
+ goto end;
}
- DBG("Rotation completed for session %s", session->name);
+ pthread_mutex_unlock(socket->lock);
}
+skip_kernel:
+end:
+ rcu_read_unlock();
- ret = 0;
+ if (!chunk_exists_on_peer) {
+ uint64_t chunk_being_archived_id;
-end_unlock_session:
- channel_rotation_info_destroy(channel_info);
- session_unlock(session);
-end_unlock_session_list:
- session_unlock_list();
- rcu_read_unlock();
-end:
- return ret;
+ chunk_status = lttng_trace_chunk_get_id(
+ session->chunk_being_archived,
+ &chunk_being_archived_id);
+ assert(chunk_status == LTTNG_TRACE_CHUNK_STATUS_OK);
+ DBG("Rotation of trace archive %" PRIu64 " of session \"%s\" is complete on all consumers",
+ chunk_being_archived_id,
+ session->name);
+ }
+ *_rotation_completed = !chunk_exists_on_peer;
+ if (ret) {
+ ret = session_reset_rotation_state(session,
+ LTTNG_ROTATION_STATE_ERROR);
+ if (ret) {
+ ERR("Failed to reset rotation state of session \"%s\"",
+ session->name);
+ }
+ }
}
/*
- * Process the rotate_pending check, called with session lock held.
+ * Check if the last rotation was completed, called with session lock held.
+ * Should only return non-zero in the event of a fatal error. Doing so will
+ * shutdown the thread.
*/
static
-int rotate_pending_relay_timer(struct ltt_session *session)
+int check_session_rotation_pending(struct ltt_session *session,
+ struct notification_thread_handle *notification_thread_handle)
{
int ret;
+ struct lttng_trace_archive_location *location;
+ enum lttng_trace_chunk_status chunk_status;
+ bool rotation_completed = false;
+ const char *archived_chunk_name;
+ uint64_t chunk_being_archived_id;
- DBG("[rotation-thread] Check rotate pending on session %" PRIu64,
- session->id);
- ret = relay_rotate_pending(session, session->rotate_count - 1);
- if (ret < 0) {
- ERR("[rotation-thread] Check relay rotate pending");
+ if (!session->chunk_being_archived) {
+ ret = 0;
goto end;
}
- if (ret == 0) {
- DBG("[rotation-thread] Rotation completed on the relay for "
- "session %" PRIu64, session->id);
- /*
- * Now we can clear the pending flag in the session. New
- * rotations can start now.
- */
- session->rotate_pending_relay = false;
- session->rotation_state = LTTNG_ROTATION_STATE_COMPLETED;
- } else if (ret == 1) {
- DBG("[rotation-thread] Rotation still pending on the relay for "
- "session %" PRIu64, session->id);
- ret = sessiond_timer_rotate_pending_start(session,
- DEFAULT_ROTATE_PENDING_RELAY_TIMER);
+
+ chunk_status = lttng_trace_chunk_get_id(session->chunk_being_archived,
+ &chunk_being_archived_id);
+ assert(chunk_status == LTTNG_TRACE_CHUNK_STATUS_OK);
+
+ DBG("Checking for pending rotation on session \"%s\", trace archive %" PRIu64,
+ session->name, chunk_being_archived_id);
+
+ /*
+ * The rotation-pending check timer of a session is launched in
+ * one-shot mode. If the rotation is incomplete, the rotation
+ * thread will re-enable the pending-check timer.
+ *
+ * The timer thread can't stop the timer itself since it is involved
+ * in the check for the timer's quiescence.
+ */
+ ret = timer_session_rotation_pending_check_stop(session);
+ if (ret) {
+ goto check_ongoing_rotation;
+ }
+
+ check_session_rotation_pending_on_consumers(session,
+ &rotation_completed);
+ if (!rotation_completed ||
+ session->rotation_state == LTTNG_ROTATION_STATE_ERROR) {
+ goto check_ongoing_rotation;
+ }
+
+ /*
+ * Now we can clear the "ONGOING" state in the session. New
+ * rotations can start now.
+ */
+ chunk_status = lttng_trace_chunk_get_name(session->chunk_being_archived,
+ &archived_chunk_name, NULL);
+ assert(chunk_status == LTTNG_TRACE_CHUNK_STATUS_OK);
+ free(session->last_archived_chunk_name);
+ session->last_archived_chunk_name = strdup(archived_chunk_name);
+ if (!session->last_archived_chunk_name) {
+ PERROR("Failed to duplicate archived chunk name");
+ }
+ session_reset_rotation_state(session, LTTNG_ROTATION_STATE_COMPLETED);
+
+ if (!session->quiet_rotation) {
+ location = session_get_trace_archive_location(session);
+ ret = notification_thread_command_session_rotation_completed(
+ notification_thread_handle,
+ session->name,
+ session->uid,
+ session->gid,
+ session->last_archived_chunk_id.value,
+ location);
+ lttng_trace_archive_location_put(location);
+ if (ret != LTTNG_OK) {
+ ERR("Failed to notify notification thread of completed rotation for session %s",
+ session->name);
+ }
+ }
+
+ ret = 0;
+check_ongoing_rotation:
+ if (session->rotation_state == LTTNG_ROTATION_STATE_ONGOING) {
+ chunk_status = lttng_trace_chunk_get_id(
+ session->chunk_being_archived,
+ &chunk_being_archived_id);
+ assert(chunk_status == LTTNG_TRACE_CHUNK_STATUS_OK);
+
+ DBG("Rotation of trace archive %" PRIu64 " is still pending for session %s",
+ chunk_being_archived_id, session->name);
+ ret = timer_session_rotation_pending_check_start(session,
+ DEFAULT_ROTATE_PENDING_TIMER);
if (ret) {
- ERR("Re-enabling rotate pending timer");
+ ERR("Failed to re-enable rotation pending timer");
ret = -1;
goto end;
}
}
- ret = 0;
-
end:
return ret;
}
-/*
- * Process the rotate_timer, called with session lock held.
- */
+/* Call with the session and session_list locks held. */
static
-int rotate_timer(struct ltt_session *session)
+int launch_session_rotation(struct ltt_session *session)
{
int ret;
+ struct lttng_rotate_session_return rotation_return;
- /*
- * Complete _at most_ one scheduled rotation on a stopped session.
- */
- if (!session->active && session->rotate_timer_enabled &&
- session->rotated_after_last_stop) {
- ret = 0;
- goto end;
- }
+ DBG("Launching scheduled time-based rotation on session \"%s\"",
+ session->name);
- /* Ignore this timer if a rotation is already in progress. */
- if (session->rotate_pending || session->rotate_pending_relay) {
- ret = 0;
- goto end;
+ ret = cmd_rotate_session(session, &rotation_return, false,
+ LTTNG_TRACE_CHUNK_COMMAND_TYPE_MOVE_TO_COMPLETED);
+ if (ret == LTTNG_OK) {
+ DBG("Scheduled time-based rotation successfully launched on session \"%s\"",
+ session->name);
+ } else {
+ /* Don't consider errors as fatal. */
+ DBG("Scheduled time-based rotation aborted for session %s: %s",
+ session->name, lttng_strerror(ret));
}
+ return 0;
+}
- DBG("[rotation-thread] Rotate timer on session %s", session->name);
+static
+int run_job(struct rotation_thread_job *job, struct ltt_session *session,
+ struct notification_thread_handle *notification_thread_handle)
+{
+ int ret;
- ret = cmd_rotate_session(session, NULL);
- if (ret == -LTTNG_ERR_ROTATION_PENDING) {
- DBG("Scheduled rotation aborted since a rotation is already in progress");
- ret = 0;
- goto end;
- } else if (ret != LTTNG_OK) {
- ERR("[rotation-thread] Automatic time-triggered rotation failed with error code %i", ret);
- ret = -1;
- goto end;
+ switch (job->type) {
+ case ROTATION_THREAD_JOB_TYPE_SCHEDULED_ROTATION:
+ ret = launch_session_rotation(session);
+ break;
+ case ROTATION_THREAD_JOB_TYPE_CHECK_PENDING_ROTATION:
+ ret = check_session_rotation_pending(session,
+ notification_thread_handle);
+ break;
+ default:
+ abort();
}
-
- ret = 0;
-
-end:
return ret;
}
static
-int handle_rotate_timer_pipe(uint32_t revents,
- struct rotation_thread_handle *handle,
- struct rotation_thread_state *state,
+int handle_job_queue(struct rotation_thread_handle *handle,
+ struct rotation_thread *state,
struct rotation_thread_timer_queue *queue)
{
int ret = 0;
- int fd = lttng_pipe_get_readfd(queue->event_pipe);
- struct ltt_session *session;
- char buf[1];
-
- if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
- ret = lttng_poll_del(&state->events, fd);
- if (ret) {
- ERR("[rotation-thread] Failed to remove consumer "
- "rotate pending pipe from poll set");
- }
- goto end;
- }
-
- ret = lttng_read(fd, buf, 1);
- if (ret != 1) {
- ERR("[rotation-thread] Failed to read from wakeup pipe (fd = %i)", fd);
- ret = -1;
- goto end;
- }
for (;;) {
- struct sessiond_rotation_timer *timer_data;
+ struct ltt_session *session;
+ struct rotation_thread_job *job;
- /*
- * Take the queue lock only to pop elements from the list.
- */
+ /* Take the queue lock only to pop an element from the list. */
pthread_mutex_lock(&queue->lock);
if (cds_list_empty(&queue->list)) {
pthread_mutex_unlock(&queue->lock);
break;
}
- timer_data = cds_list_first_entry(&queue->list,
- struct sessiond_rotation_timer, head);
- cds_list_del(&timer_data->head);
+ job = cds_list_first_entry(&queue->list,
+ typeof(*job), head);
+ cds_list_del(&job->head);
pthread_mutex_unlock(&queue->lock);
- /*
- * session lock to lookup the session ID.
- */
session_lock_list();
- session = session_find_by_id(timer_data->session_id);
+ session = job->session;
if (!session) {
- DBG("[rotation-thread] Session %" PRIu64 " not found",
- timer_data->session_id);
+ DBG("Session \"%s\" not found",
+ session->name);
/*
- * This is a non-fatal error, and we cannot report it to the
- * user (timer), so just print the error and continue the
- * processing.
+ * This is a non-fatal error, and we cannot report it to
+ * the user (timer), so just print the error and
+ * continue the processing.
+ *
+ * While the timer thread will purge pending signals for
+ * a session on the session's destruction, it is
+ * possible for a job targeting that session to have
+ * already been queued before it was destroyed.
*/
+ free(job);
+ session_put(session);
session_unlock_list();
- free(timer_data);
continue;
}
- /*
- * Take the session lock and release the session_list lock.
- */
session_lock(session);
- session_unlock_list();
-
- if (timer_data->signal == LTTNG_SESSIOND_SIG_ROTATE_PENDING) {
- ret = rotate_pending_relay_timer(session);
- } else if (timer_data->signal == LTTNG_SESSIOND_SIG_ROTATE_TIMER) {
- ret = rotate_timer(session);
- } else {
- ERR("Unknown signal in rotate timer %d", timer_data->signal);
- ret = -1;
- }
+ ret = run_job(job, session, handle->notification_thread_handle);
session_unlock(session);
- free(timer_data);
+ /* Release reference held by the job. */
+ session_put(session);
+ session_unlock_list();
+ free(job);
if (ret) {
- ERR("Error processing timer");
goto end;
}
}
return ret;
}
-int handle_condition(
- const struct lttng_condition *condition,
+static
+int handle_condition(const struct lttng_condition *condition,
const struct lttng_evaluation *evaluation,
struct notification_thread_handle *notification_thread_handle)
{
if (condition_type != LTTNG_CONDITION_TYPE_SESSION_CONSUMED_SIZE) {
ret = -1;
- ERR("[rotation-thread] Condition type and session usage type are not the same");
+ ERR("Condition type and session usage type are not the same");
goto end;
}
condition_status = lttng_condition_session_consumed_size_get_session_name(
condition, &condition_session_name);
if (condition_status != LTTNG_CONDITION_STATUS_OK) {
- ERR("[rotation-thread] Session name could not be fetched");
+ ERR("Session name could not be fetched");
ret = -1;
goto end;
}
evaluation_status = lttng_evaluation_session_consumed_size_get_consumed_size(evaluation,
&consumed);
if (evaluation_status != LTTNG_EVALUATION_STATUS_OK) {
- ERR("[rotation-thread] Failed to get evaluation");
+ ERR("Failed to get evaluation");
ret = -1;
goto end;
}
session_lock_list();
session = session_find_by_name(condition_session_name);
if (!session) {
- ret = -1;
- session_unlock_list();
- ERR("[rotation-thread] Session \"%s\" not found",
+ DBG("Failed to find session while handling notification: notification type = %s, session name = `%s`",
+ lttng_condition_type_str(condition_type),
condition_session_name);
+ /*
+ * Not a fatal error: a session can be destroyed before we get
+ * the chance to handle the notification.
+ */
+ ret = 0;
+ session_unlock_list();
goto end;
}
session_lock(session);
- session_unlock_list();
ret = unsubscribe_session_consumed_size_rotation(session,
notification_thread_handle);
if (ret) {
- goto end;
+ goto end_unlock;
}
- ret = cmd_rotate_session(session, NULL);
+ ret = cmd_rotate_session(session, NULL, false,
+ LTTNG_TRACE_CHUNK_COMMAND_TYPE_MOVE_TO_COMPLETED);
if (ret == -LTTNG_ERR_ROTATION_PENDING) {
DBG("Rotate already pending, subscribe to the next threshold value");
- ret = 0;
} else if (ret != LTTNG_OK) {
- ERR("[rotation-thread] Failed to rotate on size notification with error: %s",
+ ERR("Failed to rotate on size notification with error: %s",
lttng_strerror(ret));
ret = -1;
goto end_unlock;
consumed + session->rotate_size,
notification_thread_handle);
if (ret) {
- ERR("[rotation-thread] Failed to subscribe to session consumed size condition");
+ ERR("Failed to subscribe to session consumed size condition");
goto end_unlock;
}
ret = 0;
end_unlock:
session_unlock(session);
+ session_put(session);
+ session_unlock_list();
end:
return ret;
}
static
-int handle_notification_channel(int fd, uint32_t revents,
+int handle_notification_channel(int fd,
struct rotation_thread_handle *handle,
- struct rotation_thread_state *state)
+ struct rotation_thread *state)
{
int ret;
bool notification_pending;
status = lttng_notification_channel_has_pending_notification(
rotate_notification_channel, ¬ification_pending);
if (status != LTTNG_NOTIFICATION_CHANNEL_STATUS_OK) {
- ERR("[rotation-thread ]Error occured while checking for pending notification");
+ ERR("Error occurred while checking for pending notification");
ret = -1;
goto end;
}
return ret;
}
+static
void *thread_rotation(void *data)
{
int ret;
struct rotation_thread_handle *handle = data;
- struct rotation_thread_state state;
+ struct rotation_thread thread;
+ int queue_pipe_fd;
- DBG("[rotation-thread] Started rotation thread");
+ DBG("Started rotation thread");
+ rcu_register_thread();
+ rcu_thread_online();
+ health_register(the_health_sessiond, HEALTH_SESSIOND_TYPE_ROTATION);
+ health_code_update();
if (!handle) {
- ERR("[rotation-thread] Invalid thread context provided");
+ ERR("Invalid thread context provided");
goto end;
}
- rcu_register_thread();
- rcu_thread_online();
+ queue_pipe_fd = lttng_pipe_get_readfd(
+ handle->rotation_timer_queue->event_pipe);
- health_register(health_sessiond, HEALTH_SESSIOND_TYPE_ROTATION);
- health_code_update();
- ret = init_thread_state(handle, &state);
+ ret = init_thread_state(handle, &thread);
if (ret) {
- goto end;
+ goto error;
}
- /* Ready to handle client connections. */
- sessiond_notify_ready();
-
while (true) {
int fd_count, i;
health_poll_entry();
- DBG("[rotation-thread] Entering poll wait");
- ret = lttng_poll_wait(&state.events, -1);
- DBG("[rotation-thread] Poll wait returned (%i)", ret);
+ DBG("Entering poll wait");
+ ret = lttng_poll_wait(&thread.events, -1);
+ DBG("Poll wait returned (%i)", ret);
health_poll_exit();
if (ret < 0) {
/*
if (errno == EINTR) {
continue;
}
- ERR("[rotation-thread] Error encountered during lttng_poll_wait (%i)", ret);
+ ERR("Error encountered during lttng_poll_wait (%i)", ret);
goto error;
}
fd_count = ret;
for (i = 0; i < fd_count; i++) {
- int fd = LTTNG_POLL_GETFD(&state.events, i);
- uint32_t revents = LTTNG_POLL_GETEV(&state.events, i);
+ int fd = LTTNG_POLL_GETFD(&thread.events, i);
+ uint32_t revents = LTTNG_POLL_GETEV(&thread.events, i);
- DBG("[rotation-thread] Handling fd (%i) activity (%u)",
+ DBG("Handling fd (%i) activity (%u)",
fd, revents);
- if (fd == handle->thread_quit_pipe) {
- DBG("[rotation-thread] Quit pipe activity");
- goto exit;
- } else if (fd == lttng_pipe_get_readfd(handle->rotation_timer_queue->event_pipe)) {
- ret = handle_rotate_timer_pipe(revents,
- handle, &state, handle->rotation_timer_queue);
+ if (revents & LPOLLERR) {
+ ERR("Polling returned an error on fd %i", fd);
+ goto error;
+ }
+
+ if (fd == rotate_notification_channel->socket) {
+ ret = handle_notification_channel(fd, handle,
+ &thread);
if (ret) {
- ERR("[rotation-thread] Failed to handle rotation timer pipe event");
+ ERR("Error occurred while handling activity on notification channel socket");
goto error;
}
- } else if (fd == handle->ust32_consumer ||
- fd == handle->ust64_consumer ||
- fd == handle->kernel_consumer) {
- ret = handle_channel_rotation_pipe(fd,
- revents, handle, &state);
+ } else {
+ /* Job queue or quit pipe activity. */
+
+ /*
+ * The job queue is serviced if there is
+ * activity on the quit pipe to ensure it is
+ * flushed and all references held in the queue
+ * are released.
+ */
+ ret = handle_job_queue(handle, &thread,
+ handle->rotation_timer_queue);
if (ret) {
- ERR("[rotation-thread] Failed to handle channel rotation pipe");
+ ERR("Failed to handle rotation timer pipe event");
goto error;
}
- } else if (fd == rotate_notification_channel->socket) {
- ret = handle_notification_channel(fd, revents,
- handle, &state);
- if (ret) {
- ERR("[rotation-thread] Error occured while handling activity on notification channel socket");
- goto error;
+
+ if (fd == queue_pipe_fd) {
+ char buf;
+
+ ret = lttng_read(fd, &buf, 1);
+ if (ret != 1) {
+ ERR("Failed to read from wakeup pipe (fd = %i)", fd);
+ goto error;
+ }
+ } else {
+ DBG("Quit pipe activity");
+ goto exit;
}
}
}
}
exit:
error:
- DBG("[rotation-thread] Exit");
- fini_thread_state(&state);
- health_unregister(health_sessiond);
+ DBG("Thread exit");
+ fini_thread_state(&thread);
+end:
+ health_unregister(the_health_sessiond);
rcu_thread_offline();
rcu_unregister_thread();
-end:
return NULL;
}
+
+static
+bool shutdown_rotation_thread(void *thread_data)
+{
+ struct rotation_thread_handle *handle = thread_data;
+ const int write_fd = lttng_pipe_get_writefd(handle->quit_pipe);
+
+ return notify_thread_pipe(write_fd) == 1;
+}
+
+bool launch_rotation_thread(struct rotation_thread_handle *handle)
+{
+ struct lttng_thread *thread;
+
+ thread = lttng_thread_create("Rotation",
+ thread_rotation,
+ shutdown_rotation_thread,
+ NULL,
+ handle);
+ if (!thread) {
+ goto error;
+ }
+ lttng_thread_put(thread);
+ return true;
+error:
+ return false;
+}