1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * counter/counter-api.h
5 * LTTng Counters API, requiring counter/config.h
7 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #ifndef _LTTNG_COUNTER_API_H
11 #define _LTTNG_COUNTER_API_H
13 #include <linux/types.h>
14 #include <linux/percpu.h>
15 #include <linux/bitops.h>
16 #include <counter/counter.h>
17 #include <counter/counter-internal.h>
18 #include <wrapper/compiler_attributes.h>
19 #include <wrapper/limits.h>
22 * Using unsigned arithmetic because overflow is defined.
24 static __always_inline
int __lttng_counter_add(const struct lib_counter_config
*config
,
25 enum lib_counter_config_alloc alloc
,
26 enum lib_counter_config_sync sync
,
27 struct lib_counter
*counter
,
28 const size_t *dimension_indexes
, int64_t v
,
32 bool overflow
= false, underflow
= false;
33 struct lib_counter_layout
*layout
;
36 if (unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
38 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
41 case COUNTER_ALLOC_PER_CPU
:
42 layout
= per_cpu_ptr(counter
->percpu_counters
, smp_processor_id());
44 case COUNTER_ALLOC_GLOBAL
:
45 layout
= &counter
->global_counters
;
51 switch (config
->counter_size
) {
52 case COUNTER_SIZE_8_BIT
:
54 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
56 int8_t global_sum_step
= counter
->global_sum_step
.s8
;
60 case COUNTER_SYNC_PER_CPU
:
65 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
66 if (unlikely(n
> (int8_t) global_sum_step
))
67 move_sum
= (int8_t) global_sum_step
/ 2;
68 else if (unlikely(n
< -(int8_t) global_sum_step
))
69 move_sum
= -((int8_t) global_sum_step
/ 2);
71 res
= cmpxchg_local(int_p
, old
, n
);
75 case COUNTER_SYNC_GLOBAL
:
79 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
80 res
= cmpxchg(int_p
, old
, n
);
87 if (v
> 0 && (v
>= U8_MAX
|| n
< old
))
89 else if (v
< 0 && (v
<= -(s64
) U8_MAX
|| n
> old
))
93 case COUNTER_SIZE_16_BIT
:
95 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
97 int16_t global_sum_step
= counter
->global_sum_step
.s16
;
101 case COUNTER_SYNC_PER_CPU
:
106 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
107 if (unlikely(n
> (int16_t) global_sum_step
))
108 move_sum
= (int16_t) global_sum_step
/ 2;
109 else if (unlikely(n
< -(int16_t) global_sum_step
))
110 move_sum
= -((int16_t) global_sum_step
/ 2);
112 res
= cmpxchg_local(int_p
, old
, n
);
113 } while (old
!= res
);
116 case COUNTER_SYNC_GLOBAL
:
120 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
121 res
= cmpxchg(int_p
, old
, n
);
122 } while (old
!= res
);
128 if (v
> 0 && (v
>= U16_MAX
|| n
< old
))
130 else if (v
< 0 && (v
<= -(s64
) U16_MAX
|| n
> old
))
134 case COUNTER_SIZE_32_BIT
:
136 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
138 int32_t global_sum_step
= counter
->global_sum_step
.s32
;
142 case COUNTER_SYNC_PER_CPU
:
147 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
148 if (unlikely(n
> (int32_t) global_sum_step
))
149 move_sum
= (int32_t) global_sum_step
/ 2;
150 else if (unlikely(n
< -(int32_t) global_sum_step
))
151 move_sum
= -((int32_t) global_sum_step
/ 2);
153 res
= cmpxchg_local(int_p
, old
, n
);
154 } while (old
!= res
);
157 case COUNTER_SYNC_GLOBAL
:
161 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
162 res
= cmpxchg(int_p
, old
, n
);
163 } while (old
!= res
);
169 if (v
> 0 && (v
>= U32_MAX
|| n
< old
))
171 else if (v
< 0 && (v
<= -(s64
) U32_MAX
|| n
> old
))
175 #if BITS_PER_LONG == 64
176 case COUNTER_SIZE_64_BIT
:
178 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
180 int64_t global_sum_step
= counter
->global_sum_step
.s64
;
184 case COUNTER_SYNC_PER_CPU
:
189 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
190 if (unlikely(n
> (int64_t) global_sum_step
))
191 move_sum
= (int64_t) global_sum_step
/ 2;
192 else if (unlikely(n
< -(int64_t) global_sum_step
))
193 move_sum
= -((int64_t) global_sum_step
/ 2);
195 res
= cmpxchg_local(int_p
, old
, n
);
196 } while (old
!= res
);
199 case COUNTER_SYNC_GLOBAL
:
203 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
204 res
= cmpxchg(int_p
, old
, n
);
205 } while (old
!= res
);
211 if (v
> 0 && n
< old
)
213 else if (v
< 0 && n
> old
)
221 if (unlikely(overflow
&& !test_bit(index
, layout
->overflow_bitmap
)))
222 set_bit(index
, layout
->overflow_bitmap
);
223 else if (unlikely(underflow
&& !test_bit(index
, layout
->underflow_bitmap
)))
224 set_bit(index
, layout
->underflow_bitmap
);
226 *remainder
= move_sum
;
230 static __always_inline
int __lttng_counter_add_percpu(const struct lib_counter_config
*config
,
231 struct lib_counter
*counter
,
232 const size_t *dimension_indexes
, int64_t v
)
237 ret
= __lttng_counter_add(config
, COUNTER_ALLOC_PER_CPU
, config
->sync
,
238 counter
, dimension_indexes
, v
, &move_sum
);
241 if (unlikely(move_sum
))
242 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, COUNTER_SYNC_GLOBAL
,
243 counter
, dimension_indexes
, move_sum
, NULL
);
247 static __always_inline
int __lttng_counter_add_global(const struct lib_counter_config
*config
,
248 struct lib_counter
*counter
,
249 const size_t *dimension_indexes
, int64_t v
)
251 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, config
->sync
, counter
,
252 dimension_indexes
, v
, NULL
);
255 static __always_inline
int lttng_counter_add(const struct lib_counter_config
*config
,
256 struct lib_counter
*counter
,
257 const size_t *dimension_indexes
, int64_t v
)
259 switch (config
->alloc
) {
260 case COUNTER_ALLOC_PER_CPU
:
262 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
263 return __lttng_counter_add_percpu(config
, counter
, dimension_indexes
, v
);
264 case COUNTER_ALLOC_GLOBAL
:
265 return __lttng_counter_add_global(config
, counter
, dimension_indexes
, v
);
271 static __always_inline
int lttng_counter_inc(const struct lib_counter_config
*config
,
272 struct lib_counter
*counter
,
273 const size_t *dimension_indexes
)
275 return lttng_counter_add(config
, counter
, dimension_indexes
, 1);
278 static __always_inline
int lttng_counter_dec(const struct lib_counter_config
*config
,
279 struct lib_counter
*counter
,
280 const size_t *dimension_indexes
)
282 return lttng_counter_add(config
, counter
, dimension_indexes
, -1);
285 #endif /* _LTTNG_COUNTER_API_H */