Commit | Line | Data |
---|---|---|
9f36eaed MJ |
1 | /* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1) |
2 | * | |
2fa2d39a FG |
3 | * lttng-context-callstack.c |
4 | * | |
5 | * LTTng callstack event context. | |
6 | * | |
7 | * Copyright (C) 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
8 | * Copyright (C) 2014 Francis Giraldeau <francis.giraldeau@gmail.com> | |
9 | * | |
0bb47c89 MD |
10 | * The callstack context can be added to any kernel event. It records |
11 | * either the kernel or the userspace callstack, up to a max depth. The | |
12 | * context is a CTF sequence, such that it uses only the space required | |
13 | * for the number of callstack entries. | |
2fa2d39a | 14 | * |
0bb47c89 MD |
15 | * It allocates callstack buffers per-CPU up to 4 interrupt nesting. |
16 | * This nesting limit is the same as defined in the ring buffer. It | |
17 | * therefore uses a fixed amount of memory, proportional to the number | |
18 | * of CPUs: | |
2fa2d39a FG |
19 | * |
20 | * size = cpus * nest * depth * sizeof(unsigned long) | |
21 | * | |
3685cc80 | 22 | * Which is 4096 bytes per CPU on 64-bit host and a depth of 128. |
0bb47c89 MD |
23 | * The allocation is done at the initialization to avoid memory |
24 | * allocation overhead while tracing, using a shallow stack. | |
2fa2d39a FG |
25 | * |
26 | * The kernel callstack is recovered using save_stack_trace(), and the | |
27 | * userspace callstack uses save_stack_trace_user(). They rely on frame | |
0bb47c89 MD |
28 | * pointers. These are usually available for the kernel, but the |
29 | * compiler option -fomit-frame-pointer frequently used in popular Linux | |
30 | * distributions may cause the userspace callstack to be unreliable, and | |
31 | * is a known limitation of this approach. If frame pointers are not | |
32 | * available, it produces no error, but the callstack will be empty. We | |
33 | * still provide the feature, because it works well for runtime | |
34 | * environments having frame pointers. In the future, unwind support | |
35 | * and/or last branch record may provide a solution to this problem. | |
2fa2d39a FG |
36 | * |
37 | * The symbol name resolution is left to the trace reader. | |
38 | */ | |
39 | ||
40 | #include <linux/module.h> | |
41 | #include <linux/slab.h> | |
42 | #include <linux/sched.h> | |
43 | #include <linux/utsname.h> | |
44 | #include <linux/stacktrace.h> | |
45 | #include <linux/spinlock.h> | |
46 | #include "lttng-events.h" | |
47 | #include "wrapper/ringbuffer/backend.h" | |
48 | #include "wrapper/ringbuffer/frontend.h" | |
49 | #include "wrapper/vmalloc.h" | |
50 | #include "lttng-tracer.h" | |
51 | ||
3685cc80 | 52 | #define MAX_ENTRIES 128 |
64cc198b | 53 | |
0bb47c89 MD |
54 | enum lttng_cs_ctx_modes { |
55 | CALLSTACK_KERNEL = 0, | |
56 | CALLSTACK_USER = 1, | |
57 | NR_CALLSTACK_MODES, | |
58 | }; | |
59 | ||
60 | struct lttng_cs_dispatch { | |
64cc198b MD |
61 | struct stack_trace stack_trace; |
62 | unsigned long entries[MAX_ENTRIES]; | |
63 | }; | |
2fa2d39a FG |
64 | |
65 | struct lttng_cs { | |
0bb47c89 | 66 | struct lttng_cs_dispatch dispatch[RING_BUFFER_MAX_NESTING]; |
2fa2d39a FG |
67 | }; |
68 | ||
69 | struct field_data { | |
2fa2d39a | 70 | struct lttng_cs __percpu *cs_percpu; |
0bb47c89 | 71 | enum lttng_cs_ctx_modes mode; |
2fa2d39a FG |
72 | }; |
73 | ||
74 | struct lttng_cs_type { | |
75 | const char *name; | |
76 | const char *save_func_name; | |
77 | void (*save_func)(struct stack_trace *trace); | |
78 | }; | |
79 | ||
2fa2d39a FG |
80 | static struct lttng_cs_type cs_types[] = { |
81 | { | |
64cc198b MD |
82 | .name = "callstack_kernel", |
83 | .save_func_name = "save_stack_trace", | |
84 | .save_func = NULL, | |
2fa2d39a FG |
85 | }, |
86 | { | |
64cc198b MD |
87 | .name = "callstack_user", |
88 | .save_func_name = "save_stack_trace_user", | |
89 | .save_func = NULL, | |
2fa2d39a FG |
90 | }, |
91 | }; | |
92 | ||
93 | static | |
0bb47c89 | 94 | int init_type(enum lttng_cs_ctx_modes mode) |
2fa2d39a FG |
95 | { |
96 | unsigned long func; | |
97 | ||
98 | if (cs_types[mode].save_func) | |
99 | return 0; | |
100 | func = kallsyms_lookup_funcptr(cs_types[mode].save_func_name); | |
101 | if (!func) { | |
102 | printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n", | |
103 | cs_types[mode].save_func_name); | |
104 | return -EINVAL; | |
105 | } | |
106 | cs_types[mode].save_func = (void *) func; | |
107 | return 0; | |
108 | } | |
109 | ||
a2b0231a FD |
110 | /* Keep track of nesting inside userspace callstack context code */ |
111 | DEFINE_PER_CPU(int, callstack_user_nesting); | |
112 | ||
2fa2d39a FG |
113 | static |
114 | struct stack_trace *stack_trace_context(struct lttng_ctx_field *field, | |
115 | struct lib_ring_buffer_ctx *ctx) | |
116 | { | |
a2b0231a | 117 | int buffer_nesting, cs_user_nesting; |
2fa2d39a | 118 | struct lttng_cs *cs; |
3c1a57e8 | 119 | struct field_data *fdata = field->priv; |
2fa2d39a | 120 | |
a2b0231a FD |
121 | /* |
122 | * Do not gather the userspace callstack context when the event was | |
123 | * triggered by the userspace callstack context saving mechanism. | |
124 | */ | |
125 | cs_user_nesting = per_cpu(callstack_user_nesting, ctx->cpu); | |
126 | ||
127 | if (fdata->mode == CALLSTACK_USER && cs_user_nesting >= 1) | |
128 | return NULL; | |
129 | ||
2fa2d39a FG |
130 | /* |
131 | * get_cpu() is not required, preemption is already | |
132 | * disabled while event is written. | |
133 | * | |
134 | * max nesting is checked in lib_ring_buffer_get_cpu(). | |
135 | * Check it again as a safety net. | |
136 | */ | |
137 | cs = per_cpu_ptr(fdata->cs_percpu, ctx->cpu); | |
a2b0231a FD |
138 | buffer_nesting = per_cpu(lib_ring_buffer_nesting, ctx->cpu) - 1; |
139 | if (buffer_nesting >= RING_BUFFER_MAX_NESTING) | |
2fa2d39a | 140 | return NULL; |
a2b0231a FD |
141 | |
142 | return &cs->dispatch[buffer_nesting].stack_trace; | |
2fa2d39a FG |
143 | } |
144 | ||
145 | /* | |
146 | * In order to reserve the correct size, the callstack is computed. The | |
147 | * resulting callstack is saved to be accessed in the record step. | |
148 | */ | |
149 | static | |
150 | size_t lttng_callstack_get_size(size_t offset, struct lttng_ctx_field *field, | |
151 | struct lib_ring_buffer_ctx *ctx, | |
152 | struct lttng_channel *chan) | |
153 | { | |
2fa2d39a | 154 | struct stack_trace *trace; |
3c1a57e8 | 155 | struct field_data *fdata = field->priv; |
12c1f012 | 156 | size_t orig_offset = offset; |
2fa2d39a FG |
157 | |
158 | /* do not write data if no space is available */ | |
159 | trace = stack_trace_context(field, ctx); | |
60d51de0 | 160 | if (unlikely(!trace)) { |
12c1f012 MD |
161 | offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int)); |
162 | offset += sizeof(unsigned int); | |
163 | offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long)); | |
164 | return offset - orig_offset; | |
60d51de0 | 165 | } |
2fa2d39a FG |
166 | |
167 | /* reset stack trace, no need to clear memory */ | |
168 | trace->nr_entries = 0; | |
169 | ||
a2b0231a FD |
170 | if (fdata->mode == CALLSTACK_USER) |
171 | ++per_cpu(callstack_user_nesting, ctx->cpu); | |
172 | ||
2fa2d39a FG |
173 | /* do the real work and reserve space */ |
174 | cs_types[fdata->mode].save_func(trace); | |
a2b0231a FD |
175 | |
176 | if (fdata->mode == CALLSTACK_USER) | |
177 | per_cpu(callstack_user_nesting, ctx->cpu)--; | |
178 | ||
ea15538d MD |
179 | /* |
180 | * Remove final ULONG_MAX delimiter. If we cannot find it, add | |
181 | * our own marker to show that the stack is incomplete. This is | |
182 | * more compact for a trace. | |
183 | */ | |
184 | if (trace->nr_entries > 0 | |
185 | && trace->entries[trace->nr_entries - 1] == ULONG_MAX) { | |
186 | trace->nr_entries--; | |
187 | } | |
12c1f012 MD |
188 | offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int)); |
189 | offset += sizeof(unsigned int); | |
190 | offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long)); | |
191 | offset += sizeof(unsigned long) * trace->nr_entries; | |
ea15538d MD |
192 | /* Add our own ULONG_MAX delimiter to show incomplete stack. */ |
193 | if (trace->nr_entries == trace->max_entries) | |
12c1f012 MD |
194 | offset += sizeof(unsigned long); |
195 | return offset - orig_offset; | |
2fa2d39a FG |
196 | } |
197 | ||
198 | static | |
199 | void lttng_callstack_record(struct lttng_ctx_field *field, | |
64cc198b MD |
200 | struct lib_ring_buffer_ctx *ctx, |
201 | struct lttng_channel *chan) | |
2fa2d39a FG |
202 | { |
203 | struct stack_trace *trace = stack_trace_context(field, ctx); | |
ea15538d | 204 | unsigned int nr_seq_entries; |
2fa2d39a | 205 | |
60d51de0 MD |
206 | if (unlikely(!trace)) { |
207 | nr_seq_entries = 0; | |
208 | lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned int)); | |
209 | chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int)); | |
210 | lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long)); | |
2fa2d39a | 211 | return; |
60d51de0 | 212 | } |
2fa2d39a | 213 | lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned int)); |
ea15538d MD |
214 | nr_seq_entries = trace->nr_entries; |
215 | if (trace->nr_entries == trace->max_entries) | |
216 | nr_seq_entries++; | |
217 | chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int)); | |
2fa2d39a FG |
218 | lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long)); |
219 | chan->ops->event_write(ctx, trace->entries, | |
220 | sizeof(unsigned long) * trace->nr_entries); | |
ea15538d MD |
221 | /* Add our own ULONG_MAX delimiter to show incomplete stack. */ |
222 | if (trace->nr_entries == trace->max_entries) { | |
223 | unsigned long delim = ULONG_MAX; | |
224 | ||
225 | chan->ops->event_write(ctx, &delim, sizeof(unsigned long)); | |
226 | } | |
2fa2d39a FG |
227 | } |
228 | ||
229 | static | |
230 | void field_data_free(struct field_data *fdata) | |
231 | { | |
2fa2d39a FG |
232 | if (!fdata) |
233 | return; | |
2fa2d39a FG |
234 | free_percpu(fdata->cs_percpu); |
235 | kfree(fdata); | |
236 | } | |
237 | ||
238 | static | |
0bb47c89 | 239 | struct field_data __percpu *field_data_create(enum lttng_cs_ctx_modes mode) |
2fa2d39a FG |
240 | { |
241 | int cpu, i; | |
2fa2d39a | 242 | struct lttng_cs __percpu *cs_set; |
64cc198b | 243 | struct field_data *fdata; |
2fa2d39a | 244 | |
64cc198b | 245 | fdata = kzalloc(sizeof(*fdata), GFP_KERNEL); |
2fa2d39a FG |
246 | if (!fdata) |
247 | return NULL; | |
248 | cs_set = alloc_percpu(struct lttng_cs); | |
249 | if (!cs_set) | |
250 | goto error_alloc; | |
251 | ||
252 | fdata->cs_percpu = cs_set; | |
253 | for_each_possible_cpu(cpu) { | |
64cc198b MD |
254 | struct lttng_cs *cs; |
255 | ||
2fa2d39a FG |
256 | cs = per_cpu_ptr(cs_set, cpu); |
257 | for (i = 0; i < RING_BUFFER_MAX_NESTING; i++) { | |
0bb47c89 | 258 | struct lttng_cs_dispatch *dispatch; |
64cc198b | 259 | |
0bb47c89 MD |
260 | dispatch = &cs->dispatch[i]; |
261 | dispatch->stack_trace.entries = dispatch->entries; | |
262 | dispatch->stack_trace.max_entries = MAX_ENTRIES; | |
2fa2d39a FG |
263 | } |
264 | } | |
0bb47c89 | 265 | fdata->mode = mode; |
2fa2d39a FG |
266 | return fdata; |
267 | ||
268 | error_alloc: | |
269 | field_data_free(fdata); | |
270 | return NULL; | |
271 | } | |
272 | ||
273 | static | |
274 | void lttng_callstack_destroy(struct lttng_ctx_field *field) | |
275 | { | |
3c1a57e8 | 276 | struct field_data *fdata = field->priv; |
2fa2d39a FG |
277 | |
278 | field_data_free(fdata); | |
279 | } | |
280 | ||
281 | static | |
0bb47c89 MD |
282 | int __lttng_add_callstack_generic(struct lttng_ctx **ctx, |
283 | enum lttng_cs_ctx_modes mode) | |
2fa2d39a FG |
284 | { |
285 | const char *ctx_name = cs_types[mode].name; | |
286 | struct lttng_ctx_field *field; | |
287 | struct field_data *fdata; | |
288 | int ret; | |
289 | ||
290 | ret = init_type(mode); | |
291 | if (ret) | |
292 | return ret; | |
293 | field = lttng_append_context(ctx); | |
294 | if (!field) | |
295 | return -ENOMEM; | |
296 | if (lttng_find_context(*ctx, ctx_name)) { | |
2fa2d39a FG |
297 | ret = -EEXIST; |
298 | goto error_find; | |
299 | } | |
64cc198b | 300 | fdata = field_data_create(mode); |
2fa2d39a FG |
301 | if (!fdata) { |
302 | ret = -ENOMEM; | |
303 | goto error_create; | |
304 | } | |
305 | ||
306 | field->event_field.name = ctx_name; | |
307 | field->event_field.type.atype = atype_sequence; | |
308 | field->event_field.type.u.sequence.elem_type.atype = atype_integer; | |
309 | field->event_field.type.u.sequence.elem_type.u.basic.integer.size = sizeof(unsigned long) * CHAR_BIT; | |
310 | field->event_field.type.u.sequence.elem_type.u.basic.integer.alignment = lttng_alignof(long) * CHAR_BIT; | |
311 | field->event_field.type.u.sequence.elem_type.u.basic.integer.signedness = lttng_is_signed_type(unsigned long); | |
312 | field->event_field.type.u.sequence.elem_type.u.basic.integer.reverse_byte_order = 0; | |
313 | field->event_field.type.u.sequence.elem_type.u.basic.integer.base = 16; | |
314 | field->event_field.type.u.sequence.elem_type.u.basic.integer.encoding = lttng_encode_none; | |
315 | ||
316 | field->event_field.type.u.sequence.length_type.atype = atype_integer; | |
317 | field->event_field.type.u.sequence.length_type.u.basic.integer.size = sizeof(unsigned int) * CHAR_BIT; | |
318 | field->event_field.type.u.sequence.length_type.u.basic.integer.alignment = lttng_alignof(unsigned int) * CHAR_BIT; | |
319 | field->event_field.type.u.sequence.length_type.u.basic.integer.signedness = lttng_is_signed_type(unsigned int); | |
320 | field->event_field.type.u.sequence.length_type.u.basic.integer.reverse_byte_order = 0; | |
321 | field->event_field.type.u.sequence.length_type.u.basic.integer.base = 10; | |
322 | field->event_field.type.u.sequence.length_type.u.basic.integer.encoding = lttng_encode_none; | |
323 | ||
324 | field->get_size_arg = lttng_callstack_get_size; | |
325 | field->record = lttng_callstack_record; | |
3c1a57e8 | 326 | field->priv = fdata; |
2fa2d39a FG |
327 | field->destroy = lttng_callstack_destroy; |
328 | wrapper_vmalloc_sync_all(); | |
2fa2d39a FG |
329 | return 0; |
330 | ||
331 | error_create: | |
332 | field_data_free(fdata); | |
333 | error_find: | |
334 | lttng_remove_context_field(ctx, field); | |
335 | return ret; | |
336 | } | |
337 | ||
338 | /** | |
339 | * lttng_add_callstack_to_ctx - add callstack event context | |
340 | * | |
341 | * @ctx: the lttng_ctx pointer to initialize | |
342 | * @type: the context type | |
343 | * | |
344 | * Supported callstack type supported: | |
345 | * LTTNG_KERNEL_CONTEXT_CALLSTACK_KERNEL | |
346 | * Records the callstack of the kernel | |
347 | * LTTNG_KERNEL_CONTEXT_CALLSTACK_USER | |
348 | * Records the callstack of the userspace program (from the kernel) | |
349 | * | |
350 | * Return 0 for success, or error code. | |
351 | */ | |
352 | int lttng_add_callstack_to_ctx(struct lttng_ctx **ctx, int type) | |
353 | { | |
354 | switch (type) { | |
355 | case LTTNG_KERNEL_CONTEXT_CALLSTACK_KERNEL: | |
356 | return __lttng_add_callstack_generic(ctx, CALLSTACK_KERNEL); | |
b874d3f3 | 357 | #ifdef CONFIG_X86 |
2fa2d39a FG |
358 | case LTTNG_KERNEL_CONTEXT_CALLSTACK_USER: |
359 | return __lttng_add_callstack_generic(ctx, CALLSTACK_USER); | |
b874d3f3 | 360 | #endif |
2fa2d39a FG |
361 | default: |
362 | return -EINVAL; | |
363 | } | |
364 | } | |
365 | EXPORT_SYMBOL_GPL(lttng_add_callstack_to_ctx); |