1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * counter/counter-api.h
5 * LTTng Counters API, requiring counter/config.h
7 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #ifndef _LTTNG_COUNTER_API_H
11 #define _LTTNG_COUNTER_API_H
13 #include <linux/types.h>
14 #include <linux/percpu.h>
15 #include <linux/bitops.h>
16 #include <counter/counter.h>
17 #include <counter/counter-internal.h>
18 #include <wrapper/limits.h>
21 * Using unsigned arithmetic because overflow is defined.
23 static __always_inline
int __lttng_counter_add(const struct lib_counter_config
*config
,
24 enum lib_counter_config_alloc alloc
,
25 enum lib_counter_config_sync sync
,
26 struct lib_counter
*counter
,
27 const size_t *dimension_indexes
, int64_t v
,
31 bool overflow
= false, underflow
= false;
32 struct lib_counter_layout
*layout
;
35 if (unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
37 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
40 case COUNTER_ALLOC_PER_CPU
:
41 layout
= per_cpu_ptr(counter
->percpu_counters
, smp_processor_id());
43 case COUNTER_ALLOC_GLOBAL
:
44 layout
= &counter
->global_counters
;
50 switch (config
->counter_size
) {
51 case COUNTER_SIZE_8_BIT
:
53 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
55 int8_t global_sum_step
= counter
->global_sum_step
.s8
;
59 case COUNTER_SYNC_PER_CPU
:
64 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
65 if (unlikely(n
> (int8_t) global_sum_step
))
66 move_sum
= (int8_t) global_sum_step
/ 2;
67 else if (unlikely(n
< -(int8_t) global_sum_step
))
68 move_sum
= -((int8_t) global_sum_step
/ 2);
70 res
= cmpxchg_local(int_p
, old
, n
);
74 case COUNTER_SYNC_GLOBAL
:
78 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
79 res
= cmpxchg(int_p
, old
, n
);
86 if (v
> 0 && (v
>= U8_MAX
|| n
< old
))
88 else if (v
< 0 && (v
<= -(s64
) U8_MAX
|| n
> old
))
92 case COUNTER_SIZE_16_BIT
:
94 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
96 int16_t global_sum_step
= counter
->global_sum_step
.s16
;
100 case COUNTER_SYNC_PER_CPU
:
105 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
106 if (unlikely(n
> (int16_t) global_sum_step
))
107 move_sum
= (int16_t) global_sum_step
/ 2;
108 else if (unlikely(n
< -(int16_t) global_sum_step
))
109 move_sum
= -((int16_t) global_sum_step
/ 2);
111 res
= cmpxchg_local(int_p
, old
, n
);
112 } while (old
!= res
);
115 case COUNTER_SYNC_GLOBAL
:
119 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
120 res
= cmpxchg(int_p
, old
, n
);
121 } while (old
!= res
);
127 if (v
> 0 && (v
>= U16_MAX
|| n
< old
))
129 else if (v
< 0 && (v
<= -(s64
) U16_MAX
|| n
> old
))
133 case COUNTER_SIZE_32_BIT
:
135 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
137 int32_t global_sum_step
= counter
->global_sum_step
.s32
;
141 case COUNTER_SYNC_PER_CPU
:
146 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
147 if (unlikely(n
> (int32_t) global_sum_step
))
148 move_sum
= (int32_t) global_sum_step
/ 2;
149 else if (unlikely(n
< -(int32_t) global_sum_step
))
150 move_sum
= -((int32_t) global_sum_step
/ 2);
152 res
= cmpxchg_local(int_p
, old
, n
);
153 } while (old
!= res
);
156 case COUNTER_SYNC_GLOBAL
:
160 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
161 res
= cmpxchg(int_p
, old
, n
);
162 } while (old
!= res
);
168 if (v
> 0 && (v
>= U32_MAX
|| n
< old
))
170 else if (v
< 0 && (v
<= -(s64
) U32_MAX
|| n
> old
))
174 #if BITS_PER_LONG == 64
175 case COUNTER_SIZE_64_BIT
:
177 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
179 int64_t global_sum_step
= counter
->global_sum_step
.s64
;
183 case COUNTER_SYNC_PER_CPU
:
188 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
189 if (unlikely(n
> (int64_t) global_sum_step
))
190 move_sum
= (int64_t) global_sum_step
/ 2;
191 else if (unlikely(n
< -(int64_t) global_sum_step
))
192 move_sum
= -((int64_t) global_sum_step
/ 2);
194 res
= cmpxchg_local(int_p
, old
, n
);
195 } while (old
!= res
);
198 case COUNTER_SYNC_GLOBAL
:
202 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
203 res
= cmpxchg(int_p
, old
, n
);
204 } while (old
!= res
);
210 if (v
> 0 && n
< old
)
212 else if (v
< 0 && n
> old
)
220 if (unlikely(overflow
&& !test_bit(index
, layout
->overflow_bitmap
)))
221 set_bit(index
, layout
->overflow_bitmap
);
222 else if (unlikely(underflow
&& !test_bit(index
, layout
->underflow_bitmap
)))
223 set_bit(index
, layout
->underflow_bitmap
);
225 *remainder
= move_sum
;
229 static __always_inline
int __lttng_counter_add_percpu(const struct lib_counter_config
*config
,
230 struct lib_counter
*counter
,
231 const size_t *dimension_indexes
, int64_t v
)
236 ret
= __lttng_counter_add(config
, COUNTER_ALLOC_PER_CPU
, config
->sync
,
237 counter
, dimension_indexes
, v
, &move_sum
);
240 if (unlikely(move_sum
))
241 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, COUNTER_SYNC_GLOBAL
,
242 counter
, dimension_indexes
, move_sum
, NULL
);
246 static __always_inline
int __lttng_counter_add_global(const struct lib_counter_config
*config
,
247 struct lib_counter
*counter
,
248 const size_t *dimension_indexes
, int64_t v
)
250 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, config
->sync
, counter
,
251 dimension_indexes
, v
, NULL
);
254 static __always_inline
int lttng_counter_add(const struct lib_counter_config
*config
,
255 struct lib_counter
*counter
,
256 const size_t *dimension_indexes
, int64_t v
)
258 switch (config
->alloc
) {
259 case COUNTER_ALLOC_PER_CPU
: /* Fallthrough */
260 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
261 return __lttng_counter_add_percpu(config
, counter
, dimension_indexes
, v
);
262 case COUNTER_ALLOC_GLOBAL
:
263 return __lttng_counter_add_global(config
, counter
, dimension_indexes
, v
);
269 static __always_inline
int lttng_counter_inc(const struct lib_counter_config
*config
,
270 struct lib_counter
*counter
,
271 const size_t *dimension_indexes
)
273 return lttng_counter_add(config
, counter
, dimension_indexes
, 1);
276 static __always_inline
int lttng_counter_dec(const struct lib_counter_config
*config
,
277 struct lib_counter
*counter
,
278 const size_t *dimension_indexes
)
280 return lttng_counter_add(config
, counter
, dimension_indexes
, -1);
283 #endif /* _LTTNG_COUNTER_API_H */