Fix: coherent state not changed atomically with metadata written

commit 122c63cb4310 ("Fix: Implement RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK")
introduces a new ioctl which returns a flag indicating whether the
metadata is in consistent state at the end of the sub-buffer.

That commit is meant to address metadata consistency issues observable
in live sessions.

However, the "consistent" state is false as soon as a producer is
active (between an outermost metadata_begin/end pair). Unfortunately,
if the last "RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK" operation is
done between the last metadata printf and "end" of the transaction, the
last consistency state will be false, and the consumer daemon will never
send metadata to the relay daemon. This in turn causes a live viewer to
wait for metadata endlessly.

This issue can be reproduced by running lttng-tools:
tests/regression/tools/live/test_kernel

as root in a loop.

We observe two things:
1) the poll operation blocks when there is no more metadata to send,
   which means there is no mean to unblock when the consistency state
   changes back to "true" without producing additional metadata,

2) Even if (1) was fixed, the expectation from an ABI perspective is
   that the "coherent" state is only populated when
   RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK succeeds. Therefore,
   there is no way to let user-space know about conherency transition
   unless additional metadata is generated.

Fixing this requires to hold the metadata cache lock across the entire
production of a coherent metadata transaction. This simpler scheme is
possible because the metadata is generated in a reallocated memory area
and not directly into a ring buffer anymore. This was not the case in
earlier lttng-modules versions, when the metadata was generated directly
into a ring buffer, which explains why this simpler scheme was not
implemented.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/lttng-events.c b/lttng-events.c
index cfe2a3529de33efb0184409d4dbdbd105eb93b8a..b7c183e91f97ca4033506890a4dc9ec47e0191c1 100644 (file)
--- a/lttng-events.c
+++ b/lttng-events.c
@@ -1669,7 +1669,7 @@ int lttng_metadata_output_channel(struct lttng_metadata_stream *stream,
                        reserve_len);
        stream->transport->ops.event_commit(&ctx);
        stream->metadata_in += reserve_len;
-       if (reserve_len < len || stream->metadata_cache->producing != 0)
+       if (reserve_len < len)
                stream->coherent = false;
        else
                stream->coherent = true;
@@ -1685,18 +1685,21 @@ end:
 static
 void lttng_metadata_begin(struct lttng_session *session)
 {
-       mutex_lock(&session->metadata_cache->lock);
-       session->metadata_cache->producing++;
-       mutex_unlock(&session->metadata_cache->lock);
+       if (atomic_inc_return(&session->metadata_cache->producing) == 1)
+               mutex_lock(&session->metadata_cache->lock);
 }
 
 static
 void lttng_metadata_end(struct lttng_session *session)
 {
-       mutex_lock(&session->metadata_cache->lock);
-       WARN_ON_ONCE(!session->metadata_cache->producing);
-       session->metadata_cache->producing--;
-       mutex_unlock(&session->metadata_cache->lock);
+       WARN_ON_ONCE(!atomic_read(&session->metadata_cache->producing));
+       if (atomic_dec_return(&session->metadata_cache->producing) == 0) {
+               struct lttng_metadata_stream *stream;
+
+               mutex_unlock(&session->metadata_cache->lock);
+               list_for_each_entry(stream, &session->metadata_cache->metadata_stream, list)
+                       wake_up_interruptible(&stream->read_wait);
+       }
 }
 
 /*
@@ -1713,7 +1716,6 @@ int lttng_metadata_printf(struct lttng_session *session,
        char *str;
        size_t len;
        va_list ap;
-       struct lttng_metadata_stream *stream;
 
        WARN_ON_ONCE(!READ_ONCE(session->active));
 
@@ -1724,8 +1726,7 @@ int lttng_metadata_printf(struct lttng_session *session,
                return -ENOMEM;
 
        len = strlen(str);
-       mutex_lock(&session->metadata_cache->lock);
-       session->metadata_cache->producing++;
+       WARN_ON_ONCE(!atomic_read(&session->metadata_cache->producing));
        if (session->metadata_cache->metadata_written + len >
                        session->metadata_cache->cache_alloc) {
                char *tmp_cache_realloc;
@@ -1751,18 +1752,11 @@ int lttng_metadata_printf(struct lttng_session *session,
                        session->metadata_cache->metadata_written,
                        str, len);
        session->metadata_cache->metadata_written += len;
-       session->metadata_cache->producing--;
-       mutex_unlock(&session->metadata_cache->lock);
        kfree(str);
 
-       list_for_each_entry(stream, &session->metadata_cache->metadata_stream, list)
-               wake_up_interruptible(&stream->read_wait);
-
        return 0;
 
 err:
-       session->metadata_cache->producing--;
-       mutex_unlock(&session->metadata_cache->lock);
        kfree(str);
        return -ENOMEM;
 }

diff --git a/lttng-events.h b/lttng-events.h
index 2ee5fdcff7772d2e6d9912f179b54a14aaa7a3dd..a36a312ccb5a3e61c67d70a33bc22faf8327c8f6 100644 (file)
--- a/lttng-events.h
+++ b/lttng-events.h
@@ -549,7 +549,7 @@ struct lttng_metadata_cache {
        char *data;                     /* Metadata cache */
        unsigned int cache_alloc;       /* Metadata allocated size (bytes) */
        unsigned int metadata_written;  /* Number of bytes written in metadata cache */
-       int producing;                  /* Metadata being produced (incomplete) */
+       atomic_t producing;             /* Metadata being produced (incomplete) */
        struct kref refcount;           /* Metadata cache usage */
        struct list_head metadata_stream;       /* Metadata stream list */
        uuid_le uuid;                   /* Trace session unique ID (copy) */