Implement shadow stack for dynamic len

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/lttng-events.c b/lttng-events.c
index b748cce62c8f464f8743f5794d50ee927fa0c57e..a5fe3e865c4f9e16444917901ca1a1c4b906457f 100644 (file)
--- a/lttng-events.c
+++ b/lttng-events.c
@@ -2444,6 +2444,9 @@ static int __init lttng_events_init(void)
        if (ret)
                return ret;
        ret = wrapper_get_pageblock_flags_mask_init();
+       if (ret)
+               return ret;
+       ret = lttng_probes_init();
        if (ret)
                return ret;
        ret = lttng_context_init();

diff --git a/lttng-events.h b/lttng-events.h
index fe7628473c2afa7a7c55590379624160ebbbc0a9..56f5d3ac2a0670ac2ed4edfbc731e2aa6a8f5a59 100644 (file)
--- a/lttng-events.h
+++ b/lttng-events.h
@@ -447,6 +447,14 @@ struct lttng_metadata_stream {
        uint64_t version;               /* Current version of the metadata cache */
 };
 
+#define LTTNG_DYNAMIC_LEN_STACK_SIZE   128
+
+struct lttng_dynamic_len_stack {
+       size_t stack[LTTNG_DYNAMIC_LEN_STACK_SIZE];
+       size_t offset;
+};
+
+DECLARE_PER_CPU(struct lttng_dynamic_len_stack, lttng_dynamic_len_stack);
 
 /*
  * struct lttng_pid_tracker declared in header due to deferencing of *v
@@ -628,6 +636,8 @@ int lttng_enabler_attach_bytecode(struct lttng_enabler *enabler,
 void lttng_enabler_event_link_bytecode(struct lttng_event *event,
                struct lttng_enabler *enabler);
 
+int lttng_probes_init(void);
+
 extern struct lttng_ctx *lttng_static_ctx;
 
 int lttng_context_init(void);

diff --git a/lttng-probes.c b/lttng-probes.c
index dd2b2ce79c17f3cf6a2d986be702f2a57ebee806..51b0f8652a297eee9bb5d7a15df443972732f595 100644 (file)
--- a/lttng-probes.c
+++ b/lttng-probes.c
@@ -44,6 +44,10 @@ static LIST_HEAD(lazy_probe_init);
  */
 static int lazy_nesting;
 
+DEFINE_PER_CPU(struct lttng_dynamic_len_stack, lttng_dynamic_len_stack);
+
+EXPORT_PER_CPU_SYMBOL_GPL(lttng_dynamic_len_stack);
+
 /*
  * Called under sessions lock.
  */
@@ -322,3 +326,12 @@ const struct file_operations lttng_tracepoint_list_fops = {
        .llseek = seq_lseek,
        .release = seq_release,
 };
+
+int lttng_probes_init(void)
+{
+       int cpu;
+
+       for_each_possible_cpu(cpu)
+               per_cpu_ptr(&lttng_dynamic_len_stack, cpu)->offset = 0;
+       return 0;
+}

diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h
index ba906ac49c01fa46b34bf8e8d4a87570ab3b3a9e..42a4be30233c35d74bd56a64d25100a3a3faeb2c 100644 (file)
--- a/probes/lttng-tracepoint-event-impl.h
+++ b/probes/lttng-tracepoint-event-impl.h
@@ -370,9 +370,15 @@ static void __event_probe__##_name(void *__data);
        __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
        __event_len += sizeof(_length_type);                                   \
        __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
-       __dynamic_len[__dynamic_len_idx] = (_src_length);                      \
-       __event_len += sizeof(_type) * __dynamic_len[__dynamic_len_idx];       \
-       __dynamic_len_idx++;
+       {                                                                               \
+               size_t __seqlen = (_src_length);                                        \
+                                                                                       \
+               if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
+                       goto error;                                                     \
+               barrier();      /* reserve before use. */                               \
+               this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
+               __event_len += sizeof(_type) * __seqlen;                                \
+       }
 
 #undef _ctf_sequence_bitfield
 #define _ctf_sequence_bitfield(_type, _item, _src,             \
@@ -387,12 +393,16 @@ static void __event_probe__##_name(void *__data);
  */
 #undef _ctf_string
 #define _ctf_string(_item, _src, _user, _nowrite)                             \
-       if (_user)                                                             \
-               __event_len += __dynamic_len[__dynamic_len_idx++] =            \
+       if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
+               goto error;                                                    \
+       barrier();      /* reserve before use. */                              \
+       if (_user) {                                                           \
+               __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
                        max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
-       else                                                                   \
-               __event_len += __dynamic_len[__dynamic_len_idx++] =            \
-                       strlen(_src) + 1;
+       } else {                                                               \
+               __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
+                       strlen(_src) + 1;                                      \
+       }
 
 #undef TP_PROTO
 #define TP_PROTO(...)  __VA_ARGS__
@@ -405,8 +415,7 @@ static void __event_probe__##_name(void *__data);
 
 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
-static inline size_t __event_get_size__##_name(size_t *__dynamic_len,        \
-               void *__tp_locvar, _proto)                                    \
+static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 {                                                                            \
        size_t __event_len = 0;                                               \
        unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
@@ -414,12 +423,15 @@ static inline size_t __event_get_size__##_name(size_t *__dynamic_len,           \
                                                                              \
        _fields                                                               \
        return __event_len;                                                   \
+                                                                             \
+error:                                                                       \
+       __attribute__((unused));                                              \
+       return -1;                                                            \
 }
 
 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
-static inline size_t __event_get_size__##_name(size_t *__dynamic_len,        \
-               void *__tp_locvar)                                            \
+static inline ssize_t __event_get_size__##_name(void *__tp_locvar)           \
 {                                                                            \
        size_t __event_len = 0;                                               \
        unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
@@ -427,6 +439,10 @@ static inline size_t __event_get_size__##_name(size_t *__dynamic_len,            \
                                                                              \
        _fields                                                               \
        return __event_len;                                                   \
+                                                                             \
+error:                                                                       \
+       __attribute__((unused));                                              \
+       return -1;                                                            \
 }
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
@@ -791,7 +807,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar)        \
 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
                        _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
        {                                                               \
-               _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx]; \
+               _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
                lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
                __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
        }                                                               \
@@ -810,7 +826,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar)        \
                        _length_type, _src_length,              \
                        _user, _nowrite)                        \
        {                                                               \
-               _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
+               _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
                lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
                __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
        }                                                               \
@@ -831,7 +847,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar)        \
                        _length_type, _src_length,              \
                        _user, _nowrite)                        \
        {                                                       \
-               _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
+               _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
                lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
                __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
        }                                                               \
@@ -882,7 +898,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar)        \
 
 /* Beware: this get len actually consumes the len value */
 #undef __get_dynamic_len
-#define __get_dynamic_len(field)       __stackvar.__dynamic_len[__dynamic_len_idx++]
+#define __get_dynamic_len(field)       this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
 
 #undef TP_PROTO
 #define TP_PROTO(...)  __VA_ARGS__
@@ -933,10 +949,11 @@ static void __event_probe__##_name(void *__data, _proto)                \
        struct lttng_channel *__chan = __event->chan;                         \
        struct lttng_session *__session = __chan->session;                    \
        struct lib_ring_buffer_ctx __ctx;                                     \
-       size_t __event_len, __event_align;                                    \
-       size_t __dynamic_len_idx __attribute__((unused)) = 0;                 \
+       ssize_t __event_len;                                                  \
+       size_t __event_align;                                                 \
+       size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
        union {                                                               \
-               size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)];   \
+               size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
                char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
        } __stackvar;                                                         \
        int __ret;                                                            \
@@ -956,6 +973,8 @@ static void __event_probe__##_name(void *__data, _proto)                  \
        __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
        if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
                return;                                                       \
+       __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
+       __dynamic_len_idx = __orig_dynamic_len_offset;                        \
        _code_pre                                                             \
        if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
                struct lttng_bytecode_runtime *bc_runtime;                    \
@@ -971,8 +990,11 @@ static void __event_probe__##_name(void *__data, _proto)                 \
                if (likely(!__filter_record))                                 \
                        goto __post;                                          \
        }                                                                     \
-       __event_len = __event_get_size__##_name(__stackvar.__dynamic_len,     \
-                               tp_locvar, _args);                            \
+       __event_len = __event_get_size__##_name(tp_locvar, _args);            \
+       if (unlikely(__event_len < 0)) {                                      \
+               lib_ring_buffer_lost_event_too_big(__chan->chan);             \
+               goto __post;                                                  \
+       }                                                                     \
        __event_align = __event_get_align__##_name(tp_locvar, _args);         \
        lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
                                 __event_align, -1);                          \
@@ -983,6 +1005,8 @@ static void __event_probe__##_name(void *__data, _proto)                 \
        __chan->ops->event_commit(&__ctx);                                    \
 __post:                                                                              \
        _code_post                                                            \
+       barrier();      /* use before un-reserve. */                          \
+       this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
        return;                                                               \
 }
 
@@ -999,10 +1023,11 @@ static void __event_probe__##_name(void *__data)                       \
        struct lttng_channel *__chan = __event->chan;                         \
        struct lttng_session *__session = __chan->session;                    \
        struct lib_ring_buffer_ctx __ctx;                                     \
-       size_t __event_len, __event_align;                                    \
-       size_t __dynamic_len_idx __attribute__((unused)) = 0;                 \
+       ssize_t __event_len;                                                  \
+       size_t __event_align;                                                 \
+       size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
        union {                                                               \
-               size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)];   \
+               size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
                char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
        } __stackvar;                                                         \
        int __ret;                                                            \
@@ -1022,6 +1047,8 @@ static void __event_probe__##_name(void *__data)                        \
        __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
        if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
                return;                                                       \
+       __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
+       __dynamic_len_idx = __orig_dynamic_len_offset;                        \
        _code_pre                                                             \
        if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
                struct lttng_bytecode_runtime *bc_runtime;                    \
@@ -1037,7 +1064,11 @@ static void __event_probe__##_name(void *__data)                       \
                if (likely(!__filter_record))                                 \
                        goto __post;                                          \
        }                                                                     \
-       __event_len = __event_get_size__##_name(__stackvar.__dynamic_len, tp_locvar); \
+       __event_len = __event_get_size__##_name(tp_locvar);                   \
+       if (unlikely(__event_len < 0)) {                                      \
+               lib_ring_buffer_lost_event_too_big(__chan->chan);             \
+               goto __post;                                                  \
+       }                                                                     \
        __event_align = __event_get_align__##_name(tp_locvar);                \
        lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
                                 __event_align, -1);                          \
@@ -1048,6 +1079,8 @@ static void __event_probe__##_name(void *__data)                        \
        __chan->ops->event_commit(&__ctx);                                    \
 __post:                                                                              \
        _code_post                                                            \
+       barrier();      /* use before un-reserve. */                          \
+       this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
        return;                                                               \
 }