1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * counter/counter-api.h
5 * LTTng Counters API, requiring counter/config.h
7 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #ifndef _LTTNG_COUNTER_API_H
11 #define _LTTNG_COUNTER_API_H
13 #include <linux/types.h>
14 #include <linux/percpu.h>
15 #include <linux/bitops.h>
16 #include <counter/counter.h>
17 #include <counter/counter-internal.h>
18 #include <wrapper/limits.h>
21 * Using unsigned arithmetic because overflow is defined.
23 static inline int __lttng_counter_add(const struct lib_counter_config
*config
,
24 enum lib_counter_config_alloc alloc
,
25 enum lib_counter_config_sync sync
,
26 struct lib_counter
*counter
,
27 const size_t *dimension_indexes
, int64_t v
,
31 bool overflow
= false, underflow
= false;
32 struct lib_counter_layout
*layout
;
35 if (unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
37 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
40 case COUNTER_ALLOC_PER_CPU
:
41 layout
= per_cpu_ptr(counter
->percpu_counters
, smp_processor_id());
43 case COUNTER_ALLOC_GLOBAL
:
44 layout
= &counter
->global_counters
;
50 switch (config
->counter_size
) {
51 case COUNTER_SIZE_8_BIT
:
53 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
55 int8_t global_sum_step
= counter
->global_sum_step
.s8
;
59 case COUNTER_SYNC_PER_CPU
:
64 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
65 if (unlikely(n
> (int8_t) global_sum_step
))
66 move_sum
= (int8_t) global_sum_step
/ 2;
67 else if (unlikely(n
< -(int8_t) global_sum_step
))
68 move_sum
= -((int8_t) global_sum_step
/ 2);
70 res
= cmpxchg_local(int_p
, old
, n
);
74 case COUNTER_SYNC_GLOBAL
:
78 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
79 res
= cmpxchg(int_p
, old
, n
);
84 if (v
> 0 && (v
>= U8_MAX
|| n
< old
))
86 else if (v
< 0 && (v
<= -U8_MAX
|| n
> old
))
90 case COUNTER_SIZE_16_BIT
:
92 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
94 int16_t global_sum_step
= counter
->global_sum_step
.s16
;
98 case COUNTER_SYNC_PER_CPU
:
103 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
104 if (unlikely(n
> (int16_t) global_sum_step
))
105 move_sum
= (int16_t) global_sum_step
/ 2;
106 else if (unlikely(n
< -(int16_t) global_sum_step
))
107 move_sum
= -((int16_t) global_sum_step
/ 2);
109 res
= cmpxchg_local(int_p
, old
, n
);
110 } while (old
!= res
);
113 case COUNTER_SYNC_GLOBAL
:
117 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
118 res
= cmpxchg(int_p
, old
, n
);
119 } while (old
!= res
);
123 if (v
> 0 && (v
>= U16_MAX
|| n
< old
))
125 else if (v
< 0 && (v
<= -U16_MAX
|| n
> old
))
129 case COUNTER_SIZE_32_BIT
:
131 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
133 int32_t global_sum_step
= counter
->global_sum_step
.s32
;
137 case COUNTER_SYNC_PER_CPU
:
142 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
143 if (unlikely(n
> (int32_t) global_sum_step
))
144 move_sum
= (int32_t) global_sum_step
/ 2;
145 else if (unlikely(n
< -(int32_t) global_sum_step
))
146 move_sum
= -((int32_t) global_sum_step
/ 2);
148 res
= cmpxchg_local(int_p
, old
, n
);
149 } while (old
!= res
);
152 case COUNTER_SYNC_GLOBAL
:
156 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
157 res
= cmpxchg(int_p
, old
, n
);
158 } while (old
!= res
);
162 if (v
> 0 && (v
>= U32_MAX
|| n
< old
))
164 else if (v
< 0 && (v
<= -U32_MAX
|| n
> old
))
168 #if BITS_PER_LONG == 64
169 case COUNTER_SIZE_64_BIT
:
171 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
173 int64_t global_sum_step
= counter
->global_sum_step
.s64
;
177 case COUNTER_SYNC_PER_CPU
:
182 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
183 if (unlikely(n
> (int64_t) global_sum_step
))
184 move_sum
= (int64_t) global_sum_step
/ 2;
185 else if (unlikely(n
< -(int64_t) global_sum_step
))
186 move_sum
= -((int64_t) global_sum_step
/ 2);
188 res
= cmpxchg_local(int_p
, old
, n
);
189 } while (old
!= res
);
192 case COUNTER_SYNC_GLOBAL
:
196 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
197 res
= cmpxchg(int_p
, old
, n
);
198 } while (old
!= res
);
202 if (v
> 0 && n
< old
)
204 else if (v
< 0 && n
> old
)
212 if (unlikely(overflow
&& !test_bit(index
, layout
->overflow_bitmap
)))
213 set_bit(index
, layout
->overflow_bitmap
);
214 else if (unlikely(underflow
&& !test_bit(index
, layout
->underflow_bitmap
)))
215 set_bit(index
, layout
->underflow_bitmap
);
217 *remainder
= move_sum
;
221 static inline int __lttng_counter_add_percpu(const struct lib_counter_config
*config
,
222 struct lib_counter
*counter
,
223 const size_t *dimension_indexes
, int64_t v
)
228 ret
= __lttng_counter_add(config
, COUNTER_ALLOC_PER_CPU
, config
->sync
,
229 counter
, dimension_indexes
, v
, &move_sum
);
232 if (unlikely(move_sum
))
233 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, COUNTER_SYNC_GLOBAL
,
234 counter
, dimension_indexes
, move_sum
, NULL
);
238 static inline int __lttng_counter_add_global(const struct lib_counter_config
*config
,
239 struct lib_counter
*counter
,
240 const size_t *dimension_indexes
, int64_t v
)
242 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, config
->sync
, counter
,
243 dimension_indexes
, v
, NULL
);
246 static inline int lttng_counter_add(const struct lib_counter_config
*config
,
247 struct lib_counter
*counter
,
248 const size_t *dimension_indexes
, int64_t v
)
250 switch (config
->alloc
) {
251 case COUNTER_ALLOC_PER_CPU
: /* Fallthrough */
252 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
253 return __lttng_counter_add_percpu(config
, counter
, dimension_indexes
, v
);
254 case COUNTER_ALLOC_GLOBAL
:
255 return __lttng_counter_add_global(config
, counter
, dimension_indexes
, v
);
261 static inline int lttng_counter_inc(const struct lib_counter_config
*config
,
262 struct lib_counter
*counter
,
263 const size_t *dimension_indexes
)
265 return lttng_counter_add(config
, counter
, dimension_indexes
, 1);
268 static inline int lttng_counter_dec(const struct lib_counter_config
*config
,
269 struct lib_counter
*counter
,
270 const size_t *dimension_indexes
)
272 return lttng_counter_add(config
, counter
, dimension_indexes
, -1);
275 #endif /* _LTTNG_COUNTER_API_H */