2 * Copyright (C) 2009 Pierre-Marc Fournier
3 * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 * Do _not_ define _LGPL_SOURCE because we don't want to create a
22 * circular dependency loop between this malloc wrapper, liburcu and
25 #include <lttng/ust-dlfcn.h>
26 #include <sys/types.h>
29 #include <urcu/system.h>
30 #include <urcu/uatomic.h>
31 #include <urcu/compiler.h>
32 #include <urcu/tls-compat.h>
33 #include <urcu/arch.h>
34 #include <lttng/align.h>
37 #define TRACEPOINT_DEFINE
38 #define TRACEPOINT_CREATE_PROBES
39 #define TP_IP_PARAM ip
42 #define STATIC_CALLOC_LEN 4096
43 static char static_calloc_buf
[STATIC_CALLOC_LEN
];
44 static unsigned long static_calloc_buf_offset
;
46 struct alloc_functions
{
47 void *(*calloc
)(size_t nmemb
, size_t size
);
48 void *(*malloc
)(size_t size
);
49 void (*free
)(void *ptr
);
50 void *(*realloc
)(void *ptr
, size_t size
);
51 void *(*memalign
)(size_t alignment
, size_t size
);
52 int (*posix_memalign
)(void **memptr
, size_t alignment
, size_t size
);
56 struct alloc_functions cur_alloc
;
59 * Make sure our own use of the LTS compat layer will not cause infinite
60 * recursion by calling calloc.
64 void *static_calloc(size_t nmemb
, size_t size
);
67 * pthread mutex replacement for URCU tls compat layer.
69 static int ust_malloc_lock
;
71 static __attribute__((unused
))
72 void ust_malloc_spin_lock(pthread_mutex_t
*lock
)
75 * The memory barrier within cmpxchg takes care of ordering
76 * memory accesses with respect to the start of the critical
79 while (uatomic_cmpxchg(&ust_malloc_lock
, 0, 1) != 0)
83 static __attribute__((unused
))
84 void ust_malloc_spin_unlock(pthread_mutex_t
*lock
)
87 * Ensure memory accesses within the critical section do not
91 uatomic_set(&ust_malloc_lock
, 0);
94 #define calloc static_calloc
95 #define pthread_mutex_lock ust_malloc_spin_lock
96 #define pthread_mutex_unlock ust_malloc_spin_unlock
97 static DEFINE_URCU_TLS(int, malloc_nesting
);
98 #undef pthread_mutex_unlock
99 #undef pthread_mutex_lock
103 * Static allocator to use when initially executing dlsym(). It keeps a
104 * size_t value of each object size prior to the object.
107 void *static_calloc_aligned(size_t nmemb
, size_t size
, size_t alignment
)
109 size_t prev_offset
, new_offset
, res_offset
, aligned_offset
;
111 if (nmemb
* size
== 0) {
116 * Protect static_calloc_buf_offset from concurrent updates
117 * using a cmpxchg loop rather than a mutex to remove a
118 * dependency on pthread. This will minimize the risk of bad
119 * interaction between mutex and malloc instrumentation.
121 res_offset
= CMM_LOAD_SHARED(static_calloc_buf_offset
);
123 prev_offset
= res_offset
;
124 aligned_offset
= LTTNG_UST_ALIGN(prev_offset
+ sizeof(size_t), alignment
);
125 new_offset
= aligned_offset
+ nmemb
* size
;
126 if (new_offset
> sizeof(static_calloc_buf
)) {
129 } while ((res_offset
= uatomic_cmpxchg(&static_calloc_buf_offset
,
130 prev_offset
, new_offset
)) != prev_offset
);
131 *(size_t *) &static_calloc_buf
[aligned_offset
- sizeof(size_t)] = size
;
132 return &static_calloc_buf
[aligned_offset
];
136 void *static_calloc(size_t nmemb
, size_t size
)
140 retval
= static_calloc_aligned(nmemb
, size
, 1);
145 void *static_malloc(size_t size
)
149 retval
= static_calloc_aligned(1, size
, 1);
154 void static_free(void *ptr
)
160 void *static_realloc(void *ptr
, size_t size
)
162 size_t *old_size
= NULL
;
171 old_size
= (size_t *) ptr
- 1;
172 if (size
<= *old_size
) {
173 /* We can re-use the old entry. */
179 /* We need to expand. Don't free previous memory location. */
180 retval
= static_calloc_aligned(1, size
, 1);
183 memcpy(retval
, ptr
, *old_size
);
189 void *static_memalign(size_t alignment
, size_t size
)
193 retval
= static_calloc_aligned(1, size
, alignment
);
198 int static_posix_memalign(void **memptr
, size_t alignment
, size_t size
)
202 /* Check for power of 2, larger than void *. */
203 if (alignment
& (alignment
- 1)
204 || alignment
< sizeof(void *)
208 ptr
= static_calloc_aligned(1, size
, alignment
);
215 void setup_static_allocator(void)
217 assert(cur_alloc
.calloc
== NULL
);
218 cur_alloc
.calloc
= static_calloc
;
219 assert(cur_alloc
.malloc
== NULL
);
220 cur_alloc
.malloc
= static_malloc
;
221 assert(cur_alloc
.free
== NULL
);
222 cur_alloc
.free
= static_free
;
223 assert(cur_alloc
.realloc
== NULL
);
224 cur_alloc
.realloc
= static_realloc
;
225 assert(cur_alloc
.memalign
== NULL
);
226 cur_alloc
.memalign
= static_memalign
;
227 assert(cur_alloc
.posix_memalign
== NULL
);
228 cur_alloc
.posix_memalign
= static_posix_memalign
;
232 void lookup_all_symbols(void)
234 struct alloc_functions af
;
237 * Temporarily redirect allocation functions to
238 * static_calloc_aligned, and free function to static_free
239 * (no-op), until the dlsym lookup has completed.
241 setup_static_allocator();
243 /* Perform the actual lookups */
244 af
.calloc
= dlsym(RTLD_NEXT
, "calloc");
245 af
.malloc
= dlsym(RTLD_NEXT
, "malloc");
246 af
.free
= dlsym(RTLD_NEXT
, "free");
247 af
.realloc
= dlsym(RTLD_NEXT
, "realloc");
248 af
.memalign
= dlsym(RTLD_NEXT
, "memalign");
249 af
.posix_memalign
= dlsym(RTLD_NEXT
, "posix_memalign");
251 /* Populate the new allocator functions */
252 memcpy(&cur_alloc
, &af
, sizeof(cur_alloc
));
255 void *malloc(size_t size
)
259 URCU_TLS(malloc_nesting
)++;
260 if (cur_alloc
.malloc
== NULL
) {
261 lookup_all_symbols();
262 if (cur_alloc
.malloc
== NULL
) {
263 fprintf(stderr
, "mallocwrap: unable to find malloc\n");
267 retval
= cur_alloc
.malloc(size
);
268 if (URCU_TLS(malloc_nesting
) == 1) {
269 tracepoint(lttng_ust_libc
, malloc
,
270 size
, retval
, LTTNG_UST_CALLER_IP());
272 URCU_TLS(malloc_nesting
)--;
278 URCU_TLS(malloc_nesting
)++;
280 * Check whether the memory was allocated with
281 * static_calloc_align, in which case there is nothing to free.
283 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
284 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
288 if (URCU_TLS(malloc_nesting
) == 1) {
289 tracepoint(lttng_ust_libc
, free
,
290 ptr
, LTTNG_UST_CALLER_IP());
293 if (cur_alloc
.free
== NULL
) {
294 lookup_all_symbols();
295 if (cur_alloc
.free
== NULL
) {
296 fprintf(stderr
, "mallocwrap: unable to find free\n");
302 URCU_TLS(malloc_nesting
)--;
305 void *calloc(size_t nmemb
, size_t size
)
309 URCU_TLS(malloc_nesting
)++;
310 if (cur_alloc
.calloc
== NULL
) {
311 lookup_all_symbols();
312 if (cur_alloc
.calloc
== NULL
) {
313 fprintf(stderr
, "callocwrap: unable to find calloc\n");
317 retval
= cur_alloc
.calloc(nmemb
, size
);
318 if (URCU_TLS(malloc_nesting
) == 1) {
319 tracepoint(lttng_ust_libc
, calloc
,
320 nmemb
, size
, retval
, LTTNG_UST_CALLER_IP());
322 URCU_TLS(malloc_nesting
)--;
326 void *realloc(void *ptr
, size_t size
)
330 URCU_TLS(malloc_nesting
)++;
332 * Check whether the memory was allocated with
333 * static_calloc_align, in which case there is nothing
334 * to free, and we need to copy the old data.
336 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
337 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
340 old_size
= (size_t *) ptr
- 1;
341 if (cur_alloc
.calloc
== NULL
) {
342 lookup_all_symbols();
343 if (cur_alloc
.calloc
== NULL
) {
344 fprintf(stderr
, "reallocwrap: unable to find calloc\n");
348 retval
= cur_alloc
.calloc(1, size
);
350 memcpy(retval
, ptr
, *old_size
);
353 * Mimick that a NULL pointer has been received, so
354 * memory allocation analysis based on the trace don't
355 * get confused by the address from the static
362 if (cur_alloc
.realloc
== NULL
) {
363 lookup_all_symbols();
364 if (cur_alloc
.realloc
== NULL
) {
365 fprintf(stderr
, "reallocwrap: unable to find realloc\n");
369 retval
= cur_alloc
.realloc(ptr
, size
);
371 if (URCU_TLS(malloc_nesting
) == 1) {
372 tracepoint(lttng_ust_libc
, realloc
,
373 ptr
, size
, retval
, LTTNG_UST_CALLER_IP());
375 URCU_TLS(malloc_nesting
)--;
379 void *memalign(size_t alignment
, size_t size
)
383 URCU_TLS(malloc_nesting
)++;
384 if (cur_alloc
.memalign
== NULL
) {
385 lookup_all_symbols();
386 if (cur_alloc
.memalign
== NULL
) {
387 fprintf(stderr
, "memalignwrap: unable to find memalign\n");
391 retval
= cur_alloc
.memalign(alignment
, size
);
392 if (URCU_TLS(malloc_nesting
) == 1) {
393 tracepoint(lttng_ust_libc
, memalign
,
394 alignment
, size
, retval
,
395 LTTNG_UST_CALLER_IP());
397 URCU_TLS(malloc_nesting
)--;
401 int posix_memalign(void **memptr
, size_t alignment
, size_t size
)
405 URCU_TLS(malloc_nesting
)++;
406 if (cur_alloc
.posix_memalign
== NULL
) {
407 lookup_all_symbols();
408 if (cur_alloc
.posix_memalign
== NULL
) {
409 fprintf(stderr
, "posix_memalignwrap: unable to find posix_memalign\n");
413 retval
= cur_alloc
.posix_memalign(memptr
, alignment
, size
);
414 if (URCU_TLS(malloc_nesting
) == 1) {
415 tracepoint(lttng_ust_libc
, posix_memalign
,
416 *memptr
, alignment
, size
,
417 retval
, LTTNG_UST_CALLER_IP());
419 URCU_TLS(malloc_nesting
)--;
424 void lttng_ust_fixup_malloc_nesting_tls(void)
426 asm volatile ("" : : "m" (URCU_TLS(malloc_nesting
)));
429 __attribute__((constructor
))
430 void lttng_ust_malloc_wrapper_init(void)
432 /* Initialization already done */
433 if (cur_alloc
.calloc
) {
436 lttng_ust_fixup_malloc_nesting_tls();
438 * Ensure the allocator is in place before the process becomes
441 lookup_all_symbols();