Commit | Line | Data |
---|---|---|
a101fa10 MD |
1 | /* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) |
2 | * | |
3 | * counter.c | |
4 | * | |
5 | * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
6 | */ | |
7 | ||
8 | #include <linux/module.h> | |
9 | #include <linux/slab.h> | |
10 | #include <lttng/tracer.h> | |
11 | #include <linux/cpumask.h> | |
12 | #include <counter/counter.h> | |
13 | #include <counter/counter-internal.h> | |
14 | #include <wrapper/vmalloc.h> | |
d410bc8d | 15 | #include <wrapper/limits.h> |
a101fa10 MD |
16 | |
17 | static size_t lttng_counter_get_dimension_nr_elements(struct lib_counter_dimension *dimension) | |
18 | { | |
19 | return dimension->max_nr_elem; | |
20 | } | |
21 | ||
22 | static int lttng_counter_init_stride(const struct lib_counter_config *config, | |
23 | struct lib_counter *counter) | |
24 | { | |
25 | size_t nr_dimensions = counter->nr_dimensions; | |
26 | size_t stride = 1; | |
27 | ssize_t i; | |
28 | ||
29 | for (i = nr_dimensions - 1; i >= 0; i--) { | |
30 | struct lib_counter_dimension *dimension = &counter->dimensions[i]; | |
31 | size_t nr_elem; | |
32 | ||
33 | nr_elem = lttng_counter_get_dimension_nr_elements(dimension); | |
34 | dimension->stride = stride; | |
35 | /* nr_elem should be minimum 1 for each dimension. */ | |
36 | if (!nr_elem) | |
37 | return -EINVAL; | |
38 | stride *= nr_elem; | |
39 | if (stride > SIZE_MAX / nr_elem) | |
40 | return -EINVAL; | |
41 | } | |
42 | return 0; | |
43 | } | |
44 | ||
45 | static int lttng_counter_layout_init(struct lib_counter *counter, int cpu) | |
46 | { | |
47 | struct lib_counter_layout *layout; | |
48 | size_t counter_size; | |
49 | size_t nr_elem = counter->allocated_elem; | |
50 | ||
51 | if (cpu == -1) | |
52 | layout = &counter->global_counters; | |
53 | else | |
54 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
55 | switch (counter->config.counter_size) { | |
56 | case COUNTER_SIZE_8_BIT: | |
57 | case COUNTER_SIZE_16_BIT: | |
58 | case COUNTER_SIZE_32_BIT: | |
59 | case COUNTER_SIZE_64_BIT: | |
60 | counter_size = (size_t) counter->config.counter_size; | |
61 | break; | |
62 | default: | |
63 | return -EINVAL; | |
64 | } | |
65 | layout->counters = lttng_kvzalloc_node(ALIGN(counter_size * nr_elem, | |
66 | 1 << INTERNODE_CACHE_SHIFT), | |
67 | GFP_KERNEL | __GFP_NOWARN, | |
68 | cpu_to_node(max(cpu, 0))); | |
69 | if (!layout->counters) | |
70 | return -ENOMEM; | |
71 | layout->overflow_bitmap = lttng_kvzalloc_node(ALIGN(ALIGN(nr_elem, 8) / 8, | |
72 | 1 << INTERNODE_CACHE_SHIFT), | |
73 | GFP_KERNEL | __GFP_NOWARN, | |
74 | cpu_to_node(max(cpu, 0))); | |
75 | if (!layout->overflow_bitmap) | |
76 | return -ENOMEM; | |
77 | layout->underflow_bitmap = lttng_kvzalloc_node(ALIGN(ALIGN(nr_elem, 8) / 8, | |
78 | 1 << INTERNODE_CACHE_SHIFT), | |
79 | GFP_KERNEL | __GFP_NOWARN, | |
80 | cpu_to_node(max(cpu, 0))); | |
81 | if (!layout->underflow_bitmap) | |
82 | return -ENOMEM; | |
83 | return 0; | |
84 | } | |
85 | ||
86 | static void lttng_counter_layout_fini(struct lib_counter *counter, int cpu) | |
87 | { | |
88 | struct lib_counter_layout *layout; | |
89 | ||
90 | if (cpu == -1) | |
91 | layout = &counter->global_counters; | |
92 | else | |
93 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
94 | ||
95 | lttng_kvfree(layout->counters); | |
96 | lttng_kvfree(layout->overflow_bitmap); | |
97 | lttng_kvfree(layout->underflow_bitmap); | |
98 | } | |
99 | ||
100 | static | |
101 | int lttng_counter_set_global_sum_step(struct lib_counter *counter, | |
102 | int64_t global_sum_step) | |
103 | { | |
104 | if (global_sum_step < 0) | |
105 | return -EINVAL; | |
106 | ||
107 | switch (counter->config.counter_size) { | |
108 | case COUNTER_SIZE_8_BIT: | |
109 | if (global_sum_step > S8_MAX) | |
110 | return -EINVAL; | |
111 | counter->global_sum_step.s8 = (int8_t) global_sum_step; | |
112 | break; | |
113 | case COUNTER_SIZE_16_BIT: | |
114 | if (global_sum_step > S16_MAX) | |
115 | return -EINVAL; | |
116 | counter->global_sum_step.s16 = (int16_t) global_sum_step; | |
117 | break; | |
118 | case COUNTER_SIZE_32_BIT: | |
119 | if (global_sum_step > S32_MAX) | |
120 | return -EINVAL; | |
121 | counter->global_sum_step.s32 = (int32_t) global_sum_step; | |
122 | break; | |
123 | case COUNTER_SIZE_64_BIT: | |
124 | counter->global_sum_step.s64 = global_sum_step; | |
125 | break; | |
126 | default: | |
127 | return -EINVAL; | |
128 | } | |
129 | ||
130 | return 0; | |
131 | } | |
132 | ||
133 | static | |
134 | int validate_args(const struct lib_counter_config *config, | |
135 | size_t nr_dimensions, | |
136 | const size_t *max_nr_elem, | |
137 | int64_t global_sum_step) | |
138 | { | |
139 | if (BITS_PER_LONG != 64 && config->counter_size == COUNTER_SIZE_64_BIT) { | |
140 | WARN_ON_ONCE(1); | |
141 | return -1; | |
142 | } | |
143 | if (!max_nr_elem) | |
144 | return -1; | |
145 | /* | |
146 | * global sum step is only useful with allocating both per-cpu | |
147 | * and global counters. | |
148 | */ | |
149 | if (global_sum_step && (!(config->alloc & COUNTER_ALLOC_GLOBAL) || | |
150 | !(config->alloc & COUNTER_ALLOC_PER_CPU))) | |
151 | return -1; | |
152 | return 0; | |
153 | } | |
154 | ||
155 | struct lib_counter *lttng_counter_create(const struct lib_counter_config *config, | |
156 | size_t nr_dimensions, | |
157 | const size_t *max_nr_elem, | |
158 | int64_t global_sum_step) | |
159 | { | |
160 | struct lib_counter *counter; | |
161 | size_t dimension, nr_elem = 1; | |
162 | int cpu, ret; | |
163 | ||
164 | if (validate_args(config, nr_dimensions, max_nr_elem, global_sum_step)) | |
165 | return NULL; | |
166 | counter = kzalloc(sizeof(struct lib_counter), GFP_KERNEL); | |
167 | if (!counter) | |
168 | return NULL; | |
169 | counter->config = *config; | |
170 | if (lttng_counter_set_global_sum_step(counter, global_sum_step)) | |
171 | goto error_sum_step; | |
172 | counter->nr_dimensions = nr_dimensions; | |
173 | counter->dimensions = kzalloc(nr_dimensions * sizeof(*counter->dimensions), GFP_KERNEL); | |
174 | if (!counter->dimensions) | |
175 | goto error_dimensions; | |
176 | for (dimension = 0; dimension < nr_dimensions; dimension++) | |
177 | counter->dimensions[dimension].max_nr_elem = max_nr_elem[dimension]; | |
178 | if (config->alloc & COUNTER_ALLOC_PER_CPU) { | |
179 | counter->percpu_counters = alloc_percpu(struct lib_counter_layout); | |
180 | if (!counter->percpu_counters) | |
181 | goto error_alloc_percpu; | |
182 | } | |
183 | ||
184 | if (lttng_counter_init_stride(config, counter)) | |
185 | goto error_init_stride; | |
186 | //TODO saturation values. | |
187 | for (dimension = 0; dimension < counter->nr_dimensions; dimension++) | |
188 | nr_elem *= lttng_counter_get_dimension_nr_elements(&counter->dimensions[dimension]); | |
189 | counter->allocated_elem = nr_elem; | |
190 | if (config->alloc & COUNTER_ALLOC_GLOBAL) { | |
191 | ret = lttng_counter_layout_init(counter, -1); /* global */ | |
192 | if (ret) | |
193 | goto layout_init_error; | |
194 | } | |
195 | if (config->alloc & COUNTER_ALLOC_PER_CPU) { | |
196 | //TODO: integrate with CPU hotplug and online cpus | |
197 | for (cpu = 0; cpu < num_possible_cpus(); cpu++) { | |
198 | ret = lttng_counter_layout_init(counter, cpu); | |
199 | if (ret) | |
200 | goto layout_init_error; | |
201 | } | |
202 | } | |
203 | return counter; | |
204 | ||
205 | layout_init_error: | |
206 | if (config->alloc & COUNTER_ALLOC_PER_CPU) { | |
207 | for (cpu = 0; cpu < num_possible_cpus(); cpu++) | |
208 | lttng_counter_layout_fini(counter, cpu); | |
209 | } | |
210 | if (config->alloc & COUNTER_ALLOC_GLOBAL) | |
211 | lttng_counter_layout_fini(counter, -1); | |
212 | error_init_stride: | |
213 | free_percpu(counter->percpu_counters); | |
214 | error_alloc_percpu: | |
215 | kfree(counter->dimensions); | |
216 | error_dimensions: | |
217 | error_sum_step: | |
218 | kfree(counter); | |
219 | return NULL; | |
220 | } | |
221 | EXPORT_SYMBOL_GPL(lttng_counter_create); | |
222 | ||
223 | void lttng_counter_destroy(struct lib_counter *counter) | |
224 | { | |
225 | struct lib_counter_config *config = &counter->config; | |
226 | int cpu; | |
227 | ||
228 | if (config->alloc & COUNTER_ALLOC_PER_CPU) { | |
229 | for (cpu = 0; cpu < num_possible_cpus(); cpu++) | |
230 | lttng_counter_layout_fini(counter, cpu); | |
231 | free_percpu(counter->percpu_counters); | |
232 | } | |
233 | if (config->alloc & COUNTER_ALLOC_GLOBAL) | |
234 | lttng_counter_layout_fini(counter, -1); | |
235 | kfree(counter->dimensions); | |
236 | kfree(counter); | |
237 | } | |
238 | EXPORT_SYMBOL_GPL(lttng_counter_destroy); | |
239 | ||
240 | int lttng_counter_read(const struct lib_counter_config *config, | |
241 | struct lib_counter *counter, | |
242 | const size_t *dimension_indexes, | |
243 | int cpu, int64_t *value, bool *overflow, | |
244 | bool *underflow) | |
245 | { | |
246 | struct lib_counter_layout *layout; | |
247 | size_t index; | |
248 | ||
249 | if (unlikely(lttng_counter_validate_indexes(config, counter, dimension_indexes))) | |
250 | return -EOVERFLOW; | |
251 | index = lttng_counter_get_index(config, counter, dimension_indexes); | |
252 | ||
253 | switch (config->alloc) { | |
254 | case COUNTER_ALLOC_PER_CPU: | |
255 | if (cpu < 0 || cpu >= num_possible_cpus()) | |
256 | return -EINVAL; | |
257 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
258 | break; | |
259 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: | |
260 | if (cpu >= 0) { | |
261 | if (cpu >= num_possible_cpus()) | |
262 | return -EINVAL; | |
263 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
264 | } else { | |
265 | layout = &counter->global_counters; | |
266 | } | |
267 | break; | |
268 | case COUNTER_ALLOC_GLOBAL: | |
269 | if (cpu >= 0) | |
270 | return -EINVAL; | |
271 | layout = &counter->global_counters; | |
272 | break; | |
273 | default: | |
274 | return -EINVAL; | |
275 | } | |
276 | ||
277 | switch (config->counter_size) { | |
278 | case COUNTER_SIZE_8_BIT: | |
279 | { | |
280 | int8_t *int_p = (int8_t *) layout->counters + index; | |
281 | *value = (int64_t) READ_ONCE(*int_p); | |
282 | break; | |
283 | } | |
284 | case COUNTER_SIZE_16_BIT: | |
285 | { | |
286 | int16_t *int_p = (int16_t *) layout->counters + index; | |
287 | *value = (int64_t) READ_ONCE(*int_p); | |
288 | break; | |
289 | } | |
290 | case COUNTER_SIZE_32_BIT: | |
291 | { | |
292 | int32_t *int_p = (int32_t *) layout->counters + index; | |
293 | *value = (int64_t) READ_ONCE(*int_p); | |
294 | break; | |
295 | } | |
296 | #if BITS_PER_LONG == 64 | |
297 | case COUNTER_SIZE_64_BIT: | |
298 | { | |
299 | int64_t *int_p = (int64_t *) layout->counters + index; | |
300 | *value = READ_ONCE(*int_p); | |
301 | break; | |
302 | } | |
303 | #endif | |
304 | default: | |
305 | WARN_ON_ONCE(1); | |
306 | } | |
307 | *overflow = test_bit(index, layout->overflow_bitmap); | |
308 | *underflow = test_bit(index, layout->underflow_bitmap); | |
309 | return 0; | |
310 | } | |
311 | EXPORT_SYMBOL_GPL(lttng_counter_read); | |
312 | ||
313 | int lttng_counter_aggregate(const struct lib_counter_config *config, | |
314 | struct lib_counter *counter, | |
315 | const size_t *dimension_indexes, | |
316 | int64_t *value, bool *overflow, | |
317 | bool *underflow) | |
318 | { | |
319 | int cpu, ret; | |
320 | int64_t v, sum = 0; | |
321 | bool of, uf; | |
322 | ||
323 | *overflow = false; | |
324 | *underflow = false; | |
325 | ||
326 | switch (config->alloc) { | |
327 | case COUNTER_ALLOC_GLOBAL: /* Fallthrough */ | |
328 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: | |
329 | /* Read global counter. */ | |
330 | ret = lttng_counter_read(config, counter, dimension_indexes, | |
331 | -1, &v, &of, &uf); | |
332 | if (ret < 0) | |
333 | return ret; | |
334 | sum += v; | |
335 | *overflow |= of; | |
336 | *underflow |= uf; | |
337 | break; | |
338 | case COUNTER_ALLOC_PER_CPU: | |
339 | break; | |
340 | } | |
341 | ||
342 | switch (config->alloc) { | |
343 | case COUNTER_ALLOC_GLOBAL: | |
344 | break; | |
345 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: /* Fallthrough */ | |
346 | case COUNTER_ALLOC_PER_CPU: | |
347 | //TODO: integrate with CPU hotplug and online cpus | |
348 | for (cpu = 0; cpu < num_possible_cpus(); cpu++) { | |
349 | int64_t old = sum; | |
350 | ||
351 | ret = lttng_counter_read(config, counter, dimension_indexes, | |
352 | cpu, &v, &of, &uf); | |
353 | if (ret < 0) | |
354 | return ret; | |
355 | *overflow |= of; | |
356 | *underflow |= uf; | |
357 | /* Overflow is defined on unsigned types. */ | |
358 | sum = (int64_t) ((uint64_t) old + (uint64_t) v); | |
359 | if (v > 0 && sum < old) | |
360 | *overflow = true; | |
361 | else if (v < 0 && sum > old) | |
362 | *underflow = true; | |
363 | } | |
364 | break; | |
365 | default: | |
366 | return -EINVAL; | |
367 | } | |
368 | *value = sum; | |
369 | return 0; | |
370 | } | |
371 | EXPORT_SYMBOL_GPL(lttng_counter_aggregate); | |
372 | ||
373 | static | |
374 | int lttng_counter_clear_cpu(const struct lib_counter_config *config, | |
375 | struct lib_counter *counter, | |
376 | const size_t *dimension_indexes, | |
377 | int cpu) | |
378 | { | |
379 | struct lib_counter_layout *layout; | |
380 | size_t index; | |
381 | ||
382 | if (unlikely(lttng_counter_validate_indexes(config, counter, dimension_indexes))) | |
383 | return -EOVERFLOW; | |
384 | index = lttng_counter_get_index(config, counter, dimension_indexes); | |
385 | ||
386 | switch (config->alloc) { | |
387 | case COUNTER_ALLOC_PER_CPU: | |
388 | if (cpu < 0 || cpu >= num_possible_cpus()) | |
389 | return -EINVAL; | |
390 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
391 | break; | |
392 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: | |
393 | if (cpu >= 0) { | |
394 | if (cpu >= num_possible_cpus()) | |
395 | return -EINVAL; | |
396 | layout = per_cpu_ptr(counter->percpu_counters, cpu); | |
397 | } else { | |
398 | layout = &counter->global_counters; | |
399 | } | |
400 | break; | |
401 | case COUNTER_ALLOC_GLOBAL: | |
402 | if (cpu >= 0) | |
403 | return -EINVAL; | |
404 | layout = &counter->global_counters; | |
405 | break; | |
406 | default: | |
407 | return -EINVAL; | |
408 | } | |
409 | switch (config->counter_size) { | |
410 | case COUNTER_SIZE_8_BIT: | |
411 | { | |
412 | int8_t *int_p = (int8_t *) layout->counters + index; | |
413 | WRITE_ONCE(*int_p, 0); | |
414 | break; | |
415 | } | |
416 | case COUNTER_SIZE_16_BIT: | |
417 | { | |
418 | int16_t *int_p = (int16_t *) layout->counters + index; | |
419 | WRITE_ONCE(*int_p, 0); | |
420 | break; | |
421 | } | |
422 | case COUNTER_SIZE_32_BIT: | |
423 | { | |
424 | int32_t *int_p = (int32_t *) layout->counters + index; | |
425 | WRITE_ONCE(*int_p, 0); | |
426 | break; | |
427 | } | |
428 | #if BITS_PER_LONG == 64 | |
429 | case COUNTER_SIZE_64_BIT: | |
430 | { | |
431 | int64_t *int_p = (int64_t *) layout->counters + index; | |
432 | WRITE_ONCE(*int_p, 0); | |
433 | break; | |
434 | } | |
435 | #endif | |
436 | default: | |
437 | WARN_ON_ONCE(1); | |
438 | } | |
439 | clear_bit(index, layout->overflow_bitmap); | |
440 | clear_bit(index, layout->underflow_bitmap); | |
441 | return 0; | |
442 | } | |
443 | ||
444 | int lttng_counter_clear(const struct lib_counter_config *config, | |
445 | struct lib_counter *counter, | |
446 | const size_t *dimension_indexes) | |
447 | { | |
448 | int cpu, ret; | |
449 | ||
450 | switch (config->alloc) { | |
451 | case COUNTER_ALLOC_GLOBAL: /* Fallthrough */ | |
452 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: | |
453 | /* Clear global counter. */ | |
454 | ret = lttng_counter_clear_cpu(config, counter, dimension_indexes, -1); | |
455 | if (ret < 0) | |
456 | return ret; | |
457 | break; | |
458 | case COUNTER_ALLOC_PER_CPU: | |
459 | break; | |
460 | } | |
461 | ||
462 | switch (config->alloc) { | |
463 | case COUNTER_ALLOC_GLOBAL: | |
464 | break; | |
465 | case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL: /* Fallthrough */ | |
466 | case COUNTER_ALLOC_PER_CPU: | |
467 | //TODO: integrate with CPU hotplug and online cpus | |
468 | for (cpu = 0; cpu < num_possible_cpus(); cpu++) { | |
469 | ret = lttng_counter_clear_cpu(config, counter, dimension_indexes, cpu); | |
470 | if (ret < 0) | |
471 | return ret; | |
472 | } | |
473 | break; | |
474 | default: | |
475 | return -EINVAL; | |
476 | } | |
477 | return 0; | |
478 | } | |
479 | EXPORT_SYMBOL_GPL(lttng_counter_clear); | |
480 | ||
481 | int lttng_counter_get_nr_dimensions(const struct lib_counter_config *config, | |
482 | struct lib_counter *counter, | |
483 | size_t *nr_dimensions) | |
484 | { | |
485 | *nr_dimensions = counter->nr_dimensions; | |
486 | return 0; | |
487 | } | |
488 | EXPORT_SYMBOL_GPL(lttng_counter_get_nr_dimensions); | |
489 | ||
490 | int lttng_counter_get_max_nr_elem(const struct lib_counter_config *config, | |
491 | struct lib_counter *counter, | |
492 | size_t *max_nr_elem) /* array of size nr_dimensions */ | |
493 | { | |
494 | size_t dimension; | |
495 | ||
496 | for (dimension = 0; dimension < counter->nr_dimensions; dimension++) | |
497 | max_nr_elem[dimension] = lttng_counter_get_dimension_nr_elements(&counter->dimensions[dimension]); | |
498 | return 0; | |
499 | } | |
500 | EXPORT_SYMBOL_GPL(lttng_counter_get_max_nr_elem); | |
501 | ||
502 | MODULE_LICENSE("GPL and additional rights"); | |
503 | MODULE_AUTHOR("Mathieu Desnoyers <mathieu.desnoyers@efficios.com>"); | |
504 | MODULE_DESCRIPTION("LTTng counter library"); | |
505 | MODULE_VERSION(__stringify(LTTNG_MODULES_MAJOR_VERSION) "." | |
506 | __stringify(LTTNG_MODULES_MINOR_VERSION) "." | |
507 | __stringify(LTTNG_MODULES_PATCHLEVEL_VERSION) | |
508 | LTTNG_MODULES_EXTRAVERSION); |