1 /* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
5 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #include "counter-internal.h"
11 #include <urcu/system.h>
12 #include <urcu/compiler.h>
14 #include <ust-helper.h>
17 #include "ust-compat.h"
19 #include "ust-bitmap.h"
21 static size_t lttng_counter_get_dimension_nr_elements(struct lib_counter_dimension
*dimension
)
23 return dimension
->max_nr_elem
;
26 static int lttng_counter_init_stride(const struct lib_counter_config
*config
,
27 struct lib_counter
*counter
)
29 size_t nr_dimensions
= counter
->nr_dimensions
;
33 for (i
= nr_dimensions
- 1; i
>= 0; i
--) {
34 struct lib_counter_dimension
*dimension
= &counter
->dimensions
[i
];
37 nr_elem
= lttng_counter_get_dimension_nr_elements(dimension
);
38 dimension
->stride
= stride
;
39 /* nr_elem should be minimum 1 for each dimension. */
43 if (stride
> SIZE_MAX
/ nr_elem
)
49 static int lttng_counter_layout_init(struct lib_counter
*counter
, int cpu
, int shm_fd
)
51 struct lib_counter_layout
*layout
;
53 size_t nr_elem
= counter
->allocated_elem
;
54 size_t shm_length
= 0, counters_offset
, overflow_offset
, underflow_offset
;
55 struct lttng_counter_shm_object
*shm_object
;
58 return 0; /* Skip, will be populated later. */
61 layout
= &counter
->global_counters
;
63 layout
= &counter
->percpu_counters
[cpu
];
64 switch (counter
->config
.counter_size
) {
65 case COUNTER_SIZE_8_BIT
:
66 case COUNTER_SIZE_16_BIT
:
67 case COUNTER_SIZE_32_BIT
:
68 case COUNTER_SIZE_64_BIT
:
69 counter_size
= (size_t) counter
->config
.counter_size
;
74 layout
->shm_fd
= shm_fd
;
75 counters_offset
= shm_length
;
76 shm_length
+= counter_size
* nr_elem
;
77 overflow_offset
= shm_length
;
78 shm_length
+= LTTNG_UST_ALIGN(nr_elem
, 8) / 8;
79 underflow_offset
= shm_length
;
80 shm_length
+= LTTNG_UST_ALIGN(nr_elem
, 8) / 8;
81 layout
->shm_len
= shm_length
;
82 if (counter
->is_daemon
) {
83 /* Allocate and clear shared memory. */
84 shm_object
= lttng_counter_shm_object_table_alloc(counter
->object_table
,
85 shm_length
, LTTNG_COUNTER_SHM_OBJECT_SHM
, shm_fd
, cpu
);
89 /* Map pre-existing shared memory. */
90 shm_object
= lttng_counter_shm_object_table_append_shm(counter
->object_table
,
95 layout
->counters
= shm_object
->memory_map
+ counters_offset
;
96 layout
->overflow_bitmap
= (unsigned long *)(shm_object
->memory_map
+ overflow_offset
);
97 layout
->underflow_bitmap
= (unsigned long *)(shm_object
->memory_map
+ underflow_offset
);
101 int lttng_counter_set_global_shm(struct lib_counter
*counter
, int fd
)
103 struct lib_counter_config
*config
= &counter
->config
;
104 struct lib_counter_layout
*layout
;
106 if (!(config
->alloc
& COUNTER_ALLOC_GLOBAL
))
108 layout
= &counter
->global_counters
;
109 if (layout
->shm_fd
>= 0)
111 return lttng_counter_layout_init(counter
, -1, fd
);
114 int lttng_counter_set_cpu_shm(struct lib_counter
*counter
, int cpu
, int fd
)
116 struct lib_counter_config
*config
= &counter
->config
;
117 struct lib_counter_layout
*layout
;
119 if (cpu
< 0 || cpu
>= lttng_counter_num_possible_cpus())
122 if (!(config
->alloc
& COUNTER_ALLOC_PER_CPU
))
124 layout
= &counter
->percpu_counters
[cpu
];
125 if (layout
->shm_fd
>= 0)
127 return lttng_counter_layout_init(counter
, cpu
, fd
);
131 int lttng_counter_set_global_sum_step(struct lib_counter
*counter
,
132 int64_t global_sum_step
)
134 if (global_sum_step
< 0)
137 switch (counter
->config
.counter_size
) {
138 case COUNTER_SIZE_8_BIT
:
139 if (global_sum_step
> INT8_MAX
)
141 counter
->global_sum_step
.s8
= (int8_t) global_sum_step
;
143 case COUNTER_SIZE_16_BIT
:
144 if (global_sum_step
> INT16_MAX
)
146 counter
->global_sum_step
.s16
= (int16_t) global_sum_step
;
148 case COUNTER_SIZE_32_BIT
:
149 if (global_sum_step
> INT32_MAX
)
151 counter
->global_sum_step
.s32
= (int32_t) global_sum_step
;
153 case COUNTER_SIZE_64_BIT
:
154 counter
->global_sum_step
.s64
= global_sum_step
;
164 int validate_args(const struct lib_counter_config
*config
,
165 size_t nr_dimensions
,
166 const size_t *max_nr_elem
,
167 int64_t global_sum_step
,
168 int global_counter_fd
,
169 int nr_counter_cpu_fds
,
170 const int *counter_cpu_fds
)
172 int nr_cpus
= lttng_counter_num_possible_cpus();
174 if (CAA_BITS_PER_LONG
!= 64 && config
->counter_size
== COUNTER_SIZE_64_BIT
) {
181 * global sum step is only useful with allocating both per-cpu
182 * and global counters.
184 if (global_sum_step
&& (!(config
->alloc
& COUNTER_ALLOC_GLOBAL
) ||
185 !(config
->alloc
& COUNTER_ALLOC_PER_CPU
)))
187 if (!(config
->alloc
& COUNTER_ALLOC_GLOBAL
) && global_counter_fd
>= 0)
189 if (!(config
->alloc
& COUNTER_ALLOC_PER_CPU
) && counter_cpu_fds
)
191 if (!(config
->alloc
& COUNTER_ALLOC_PER_CPU
) && nr_counter_cpu_fds
>= 0)
193 if (counter_cpu_fds
&& nr_cpus
!= nr_counter_cpu_fds
)
198 struct lib_counter
*lttng_counter_create(const struct lib_counter_config
*config
,
199 size_t nr_dimensions
,
200 const size_t *max_nr_elem
,
201 int64_t global_sum_step
,
202 int global_counter_fd
,
203 int nr_counter_cpu_fds
,
204 const int *counter_cpu_fds
,
207 struct lib_counter
*counter
;
208 size_t dimension
, nr_elem
= 1;
211 int nr_cpus
= lttng_counter_num_possible_cpus();
213 if (validate_args(config
, nr_dimensions
, max_nr_elem
,
214 global_sum_step
, global_counter_fd
, nr_counter_cpu_fds
,
217 counter
= zmalloc(sizeof(struct lib_counter
));
220 counter
->global_counters
.shm_fd
= -1;
221 counter
->config
= *config
;
222 counter
->is_daemon
= is_daemon
;
223 if (lttng_counter_set_global_sum_step(counter
, global_sum_step
))
225 counter
->nr_dimensions
= nr_dimensions
;
226 counter
->dimensions
= zmalloc(nr_dimensions
* sizeof(*counter
->dimensions
));
227 if (!counter
->dimensions
)
228 goto error_dimensions
;
229 for (dimension
= 0; dimension
< nr_dimensions
; dimension
++)
230 counter
->dimensions
[dimension
].max_nr_elem
= max_nr_elem
[dimension
];
231 if (config
->alloc
& COUNTER_ALLOC_PER_CPU
) {
232 counter
->percpu_counters
= zmalloc(sizeof(struct lib_counter_layout
) * nr_cpus
);
233 if (!counter
->percpu_counters
)
234 goto error_alloc_percpu
;
235 lttng_counter_for_each_possible_cpu(cpu
)
236 counter
->percpu_counters
[cpu
].shm_fd
= -1;
239 if (lttng_counter_init_stride(config
, counter
))
240 goto error_init_stride
;
241 //TODO saturation values.
242 for (dimension
= 0; dimension
< counter
->nr_dimensions
; dimension
++)
243 nr_elem
*= lttng_counter_get_dimension_nr_elements(&counter
->dimensions
[dimension
]);
244 counter
->allocated_elem
= nr_elem
;
246 if (config
->alloc
& COUNTER_ALLOC_GLOBAL
)
248 if (config
->alloc
& COUNTER_ALLOC_PER_CPU
)
249 nr_handles
+= nr_cpus
;
250 /* Allocate table for global and per-cpu counters. */
251 counter
->object_table
= lttng_counter_shm_object_table_create(nr_handles
);
252 if (!counter
->object_table
)
253 goto error_alloc_object_table
;
255 if (config
->alloc
& COUNTER_ALLOC_GLOBAL
) {
256 ret
= lttng_counter_layout_init(counter
, -1, global_counter_fd
); /* global */
258 goto layout_init_error
;
260 if ((config
->alloc
& COUNTER_ALLOC_PER_CPU
) && counter_cpu_fds
) {
261 lttng_counter_for_each_possible_cpu(cpu
) {
262 ret
= lttng_counter_layout_init(counter
, cpu
, counter_cpu_fds
[cpu
]);
264 goto layout_init_error
;
270 lttng_counter_shm_object_table_destroy(counter
->object_table
, is_daemon
);
271 error_alloc_object_table
:
273 free(counter
->percpu_counters
);
275 free(counter
->dimensions
);
282 void lttng_counter_destroy(struct lib_counter
*counter
)
284 struct lib_counter_config
*config
= &counter
->config
;
286 if (config
->alloc
& COUNTER_ALLOC_PER_CPU
)
287 free(counter
->percpu_counters
);
288 lttng_counter_shm_object_table_destroy(counter
->object_table
, counter
->is_daemon
);
289 free(counter
->dimensions
);
293 int lttng_counter_get_global_shm(struct lib_counter
*counter
, int *fd
, size_t *len
)
297 shm_fd
= counter
->global_counters
.shm_fd
;
301 *len
= counter
->global_counters
.shm_len
;
305 int lttng_counter_get_cpu_shm(struct lib_counter
*counter
, int cpu
, int *fd
, size_t *len
)
307 struct lib_counter_layout
*layout
;
310 if (cpu
>= lttng_counter_num_possible_cpus())
312 layout
= &counter
->percpu_counters
[cpu
];
313 shm_fd
= layout
->shm_fd
;
317 *len
= layout
->shm_len
;
321 int lttng_counter_read(const struct lib_counter_config
*config
,
322 struct lib_counter
*counter
,
323 const size_t *dimension_indexes
,
324 int cpu
, int64_t *value
, bool *overflow
,
328 struct lib_counter_layout
*layout
;
330 if (caa_unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
332 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
334 switch (config
->alloc
) {
335 case COUNTER_ALLOC_PER_CPU
:
336 if (cpu
< 0 || cpu
>= lttng_counter_num_possible_cpus())
338 layout
= &counter
->percpu_counters
[cpu
];
340 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
342 if (cpu
>= lttng_counter_num_possible_cpus())
344 layout
= &counter
->percpu_counters
[cpu
];
346 layout
= &counter
->global_counters
;
349 case COUNTER_ALLOC_GLOBAL
:
352 layout
= &counter
->global_counters
;
357 if (caa_unlikely(!layout
->counters
))
360 switch (config
->counter_size
) {
361 case COUNTER_SIZE_8_BIT
:
363 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
364 *value
= (int64_t) CMM_LOAD_SHARED(*int_p
);
367 case COUNTER_SIZE_16_BIT
:
369 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
370 *value
= (int64_t) CMM_LOAD_SHARED(*int_p
);
373 case COUNTER_SIZE_32_BIT
:
375 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
376 *value
= (int64_t) CMM_LOAD_SHARED(*int_p
);
379 #if CAA_BITS_PER_LONG == 64
380 case COUNTER_SIZE_64_BIT
:
382 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
383 *value
= CMM_LOAD_SHARED(*int_p
);
390 *overflow
= lttng_bitmap_test_bit(index
, layout
->overflow_bitmap
);
391 *underflow
= lttng_bitmap_test_bit(index
, layout
->underflow_bitmap
);
395 int lttng_counter_aggregate(const struct lib_counter_config
*config
,
396 struct lib_counter
*counter
,
397 const size_t *dimension_indexes
,
398 int64_t *value
, bool *overflow
,
408 switch (config
->alloc
) {
409 case COUNTER_ALLOC_GLOBAL
: /* Fallthrough */
410 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
411 /* Read global counter. */
412 ret
= lttng_counter_read(config
, counter
, dimension_indexes
,
420 case COUNTER_ALLOC_PER_CPU
:
426 switch (config
->alloc
) {
427 case COUNTER_ALLOC_GLOBAL
:
429 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
: /* Fallthrough */
430 case COUNTER_ALLOC_PER_CPU
:
431 lttng_counter_for_each_possible_cpu(cpu
) {
434 ret
= lttng_counter_read(config
, counter
, dimension_indexes
,
440 /* Overflow is defined on unsigned types. */
441 sum
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
442 if (v
> 0 && sum
< old
)
444 else if (v
< 0 && sum
> old
)
456 int lttng_counter_clear_cpu(const struct lib_counter_config
*config
,
457 struct lib_counter
*counter
,
458 const size_t *dimension_indexes
,
462 struct lib_counter_layout
*layout
;
464 if (caa_unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
466 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
468 switch (config
->alloc
) {
469 case COUNTER_ALLOC_PER_CPU
:
470 if (cpu
< 0 || cpu
>= lttng_counter_num_possible_cpus())
472 layout
= &counter
->percpu_counters
[cpu
];
474 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
476 if (cpu
>= lttng_counter_num_possible_cpus())
478 layout
= &counter
->percpu_counters
[cpu
];
480 layout
= &counter
->global_counters
;
483 case COUNTER_ALLOC_GLOBAL
:
486 layout
= &counter
->global_counters
;
491 if (caa_unlikely(!layout
->counters
))
494 switch (config
->counter_size
) {
495 case COUNTER_SIZE_8_BIT
:
497 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
498 CMM_STORE_SHARED(*int_p
, 0);
501 case COUNTER_SIZE_16_BIT
:
503 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
504 CMM_STORE_SHARED(*int_p
, 0);
507 case COUNTER_SIZE_32_BIT
:
509 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
510 CMM_STORE_SHARED(*int_p
, 0);
513 #if CAA_BITS_PER_LONG == 64
514 case COUNTER_SIZE_64_BIT
:
516 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
517 CMM_STORE_SHARED(*int_p
, 0);
524 lttng_bitmap_clear_bit(index
, layout
->overflow_bitmap
);
525 lttng_bitmap_clear_bit(index
, layout
->underflow_bitmap
);
529 int lttng_counter_clear(const struct lib_counter_config
*config
,
530 struct lib_counter
*counter
,
531 const size_t *dimension_indexes
)
535 switch (config
->alloc
) {
536 case COUNTER_ALLOC_PER_CPU
:
538 case COUNTER_ALLOC_GLOBAL
: /* Fallthrough */
539 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
540 /* Clear global counter. */
541 ret
= lttng_counter_clear_cpu(config
, counter
, dimension_indexes
, -1);
549 switch (config
->alloc
) {
550 case COUNTER_ALLOC_PER_CPU
: /* Fallthrough */
551 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
552 lttng_counter_for_each_possible_cpu(cpu
) {
553 ret
= lttng_counter_clear_cpu(config
, counter
, dimension_indexes
, cpu
);
558 case COUNTER_ALLOC_GLOBAL
: