1 #ifndef _LTTNG_WRAPPER_VMALLOC_H
2 #define _LTTNG_WRAPPER_VMALLOC_H
7 * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
8 * available, else we need to have a kernel that exports this function to GPL
11 * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
13 * This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU Lesser General Public
15 * License as published by the Free Software Foundation; only
16 * version 2.1 of the License.
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 * Lesser General Public License for more details.
23 * You should have received a copy of the GNU Lesser General Public
24 * License along with this library; if not, write to the Free Software
25 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
28 #include <linux/version.h>
29 #include <linux/vmalloc.h>
32 #ifdef CONFIG_KALLSYMS
34 #include <linux/kallsyms.h>
35 #include <wrapper/kallsyms.h>
38 void wrapper_vmalloc_sync_all(void)
40 void (*vmalloc_sync_all_sym
)(void);
42 vmalloc_sync_all_sym
= (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
43 if (vmalloc_sync_all_sym
) {
44 vmalloc_sync_all_sym();
48 * Only x86 needs vmalloc_sync_all to make sure LTTng does not
49 * trigger recursive page faults.
51 printk_once(KERN_WARNING
"LTTng: vmalloc_sync_all symbol lookup failed.\n");
52 printk_once(KERN_WARNING
"Page fault handler and NMI tracing might trigger faults.\n");
59 void wrapper_vmalloc_sync_all(void)
61 return vmalloc_sync_all();
65 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
67 void *lttng_kvmalloc_node(unsigned long size
, gfp_t flags
, int node
)
71 ret
= kvmalloc_node(size
, flags
, node
);
72 if (is_vmalloc_addr(ret
)) {
74 * Make sure we don't trigger recursive page faults in the
77 wrapper_vmalloc_sync_all();
83 void *lttng_kvzalloc_node(unsigned long size
, gfp_t flags
, int node
)
85 return lttng_kvmalloc_node(size
, flags
| __GFP_ZERO
, node
);
89 void *lttng_kvmalloc(unsigned long size
, gfp_t flags
)
91 return lttng_kvmalloc_node(size
, flags
, NUMA_NO_NODE
);
95 void *lttng_kvzalloc(unsigned long size
, gfp_t flags
)
97 return lttng_kvzalloc_node(size
, flags
, NUMA_NO_NODE
);
101 void lttng_kvfree(const void *addr
)
108 #include <linux/slab.h>
111 void print_vmalloc_node_range_warning(void)
113 printk_once(KERN_WARNING
"LTTng: __vmalloc_node_range symbol lookup failed.\n");
114 printk_once(KERN_WARNING
"Tracer performance will be degraded on NUMA systems.\n");
115 printk_once(KERN_WARNING
"Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
119 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
122 void *__lttng_vmalloc_node_range(unsigned long size
, unsigned long align
,
123 unsigned long start
, unsigned long end
, gfp_t gfp_mask
,
124 pgprot_t prot
, unsigned long vm_flags
, int node
,
127 #ifdef CONFIG_KALLSYMS
129 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
131 void *(*lttng__vmalloc_node_range
)(unsigned long size
, unsigned long align
,
132 unsigned long start
, unsigned long end
, gfp_t gfp_mask
,
133 pgprot_t prot
, unsigned long vm_flags
, int node
,
136 lttng__vmalloc_node_range
= (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
137 if (lttng__vmalloc_node_range
)
138 return lttng__vmalloc_node_range(size
, align
, start
, end
, gfp_mask
, prot
,
139 vm_flags
, node
, caller
);
141 if (node
!= NUMA_NO_NODE
)
142 print_vmalloc_node_range_warning();
143 return __vmalloc(size
, gfp_mask
, prot
);
147 * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
148 * failure, fall back to non-contiguous (vmalloc) allocation.
149 * @size: size of the request.
150 * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
152 * Uses kmalloc to get the memory but if the allocation fails then falls back
153 * to the vmalloc allocator. Use lttng_kvfree to free the memory.
155 * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
158 void *lttng_kvmalloc_node(unsigned long size
, gfp_t flags
, int node
)
163 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
164 * so the given set of flags has to be compatible.
166 WARN_ON_ONCE((flags
& GFP_KERNEL
) != GFP_KERNEL
);
169 * If the allocation fits in a single page, do not fallback.
171 if (size
<= PAGE_SIZE
) {
172 return kmalloc_node(size
, flags
, node
);
176 * Make sure that larger requests are not too disruptive - no OOM
177 * killer and no allocation failure warnings as we have a fallback
179 ret
= kmalloc_node(size
, flags
| __GFP_NOWARN
| __GFP_NORETRY
, node
);
181 ret
= __lttng_vmalloc_node_range(size
, 1,
182 VMALLOC_START
, VMALLOC_END
,
183 flags
| __GFP_HIGHMEM
, PAGE_KERNEL
, 0,
184 node
, __builtin_return_address(0));
186 * Make sure we don't trigger recursive page faults in the
189 wrapper_vmalloc_sync_all();
195 void *lttng_kvzalloc_node(unsigned long size
, gfp_t flags
, int node
)
197 return lttng_kvmalloc_node(size
, flags
| __GFP_ZERO
, node
);
201 void *lttng_kvmalloc(unsigned long size
, gfp_t flags
)
203 return lttng_kvmalloc_node(size
, flags
, NUMA_NO_NODE
);
207 void *lttng_kvzalloc(unsigned long size
, gfp_t flags
)
209 return lttng_kvzalloc_node(size
, flags
, NUMA_NO_NODE
);
213 void lttng_kvfree(const void *addr
)
215 if (is_vmalloc_addr(addr
)) {
223 #endif /* _LTTNG_WRAPPER_VMALLOC_H */