1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
5 * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
6 * available, else we need to have a kernel that exports this function to GPL
9 * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
12 #ifndef _LTTNG_WRAPPER_VMALLOC_H
13 #define _LTTNG_WRAPPER_VMALLOC_H
15 #include <linux/version.h>
16 #include <linux/vmalloc.h>
19 #ifdef CONFIG_KALLSYMS
21 #include <linux/kallsyms.h>
22 #include <wrapper/kallsyms.h>
23 #include <lttng/kernel-version.h>
25 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
26 || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
27 || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
28 || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
29 || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
30 || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
31 || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))
34 void wrapper_vmalloc_sync_mappings(void)
36 void (*vmalloc_sync_mappings_sym
)(void);
38 vmalloc_sync_mappings_sym
= (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
39 if (vmalloc_sync_mappings_sym
) {
40 vmalloc_sync_mappings_sym();
44 * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
45 * trigger recursive page faults.
47 printk_once(KERN_WARNING
"LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
48 printk_once(KERN_WARNING
"Page fault handler and NMI tracing might trigger faults.\n");
53 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
56 * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.7.
59 void wrapper_vmalloc_sync_mappings(void)
61 void (*vmalloc_sync_all_sym
)(void);
63 vmalloc_sync_all_sym
= (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
64 if (vmalloc_sync_all_sym
) {
65 vmalloc_sync_all_sym();
69 * Only x86 needs vmalloc_sync_all to make sure LTTng does not
70 * trigger recursive page faults.
72 printk_once(KERN_WARNING
"LTTng: vmalloc_sync_all symbol lookup failed.\n");
73 printk_once(KERN_WARNING
"Page fault handler and NMI tracing might trigger faults.\n");
78 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
82 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
83 || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
84 || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
85 || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
86 || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
87 || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
88 || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))
91 void wrapper_vmalloc_sync_mappings(void)
93 return vmalloc_sync_mappings();
96 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
99 void wrapper_vmalloc_sync_mappings(void)
101 return vmalloc_sync_all();
104 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
108 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
110 void *lttng_kvmalloc_node(unsigned long size
, gfp_t flags
, int node
)
114 ret
= kvmalloc_node(size
, flags
, node
);
115 if (is_vmalloc_addr(ret
)) {
117 * Make sure we don't trigger recursive page faults in the
120 wrapper_vmalloc_sync_mappings();
126 void *lttng_kvzalloc_node(unsigned long size
, gfp_t flags
, int node
)
128 return lttng_kvmalloc_node(size
, flags
| __GFP_ZERO
, node
);
132 void *lttng_kvmalloc(unsigned long size
, gfp_t flags
)
134 return lttng_kvmalloc_node(size
, flags
, NUMA_NO_NODE
);
138 void *lttng_kvzalloc(unsigned long size
, gfp_t flags
)
140 return lttng_kvzalloc_node(size
, flags
, NUMA_NO_NODE
);
144 void lttng_kvfree(const void *addr
)
151 #include <linux/slab.h>
154 void print_vmalloc_node_range_warning(void)
156 printk_once(KERN_WARNING
"LTTng: __vmalloc_node_range symbol lookup failed.\n");
157 printk_once(KERN_WARNING
"Tracer performance will be degraded on NUMA systems.\n");
158 printk_once(KERN_WARNING
"Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
162 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
165 void *__lttng_vmalloc_node_range(unsigned long size
, unsigned long align
,
166 unsigned long start
, unsigned long end
, gfp_t gfp_mask
,
167 pgprot_t prot
, unsigned long vm_flags
, int node
,
170 #ifdef CONFIG_KALLSYMS
172 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
174 void *(*lttng__vmalloc_node_range
)(unsigned long size
, unsigned long align
,
175 unsigned long start
, unsigned long end
, gfp_t gfp_mask
,
176 pgprot_t prot
, unsigned long vm_flags
, int node
,
179 lttng__vmalloc_node_range
= (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
180 if (lttng__vmalloc_node_range
)
181 return lttng__vmalloc_node_range(size
, align
, start
, end
, gfp_mask
, prot
,
182 vm_flags
, node
, caller
);
184 if (node
!= NUMA_NO_NODE
)
185 print_vmalloc_node_range_warning();
186 return __vmalloc(size
, gfp_mask
, prot
);
190 * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
191 * failure, fall back to non-contiguous (vmalloc) allocation.
192 * @size: size of the request.
193 * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
195 * Uses kmalloc to get the memory but if the allocation fails then falls back
196 * to the vmalloc allocator. Use lttng_kvfree to free the memory.
198 * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
201 void *lttng_kvmalloc_node(unsigned long size
, gfp_t flags
, int node
)
206 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
207 * so the given set of flags has to be compatible.
209 WARN_ON_ONCE((flags
& GFP_KERNEL
) != GFP_KERNEL
);
212 * If the allocation fits in a single page, do not fallback.
214 if (size
<= PAGE_SIZE
) {
215 return kmalloc_node(size
, flags
, node
);
219 * Make sure that larger requests are not too disruptive - no OOM
220 * killer and no allocation failure warnings as we have a fallback
222 ret
= kmalloc_node(size
, flags
| __GFP_NOWARN
| __GFP_NORETRY
, node
);
224 ret
= __lttng_vmalloc_node_range(size
, 1,
225 VMALLOC_START
, VMALLOC_END
,
226 flags
| __GFP_HIGHMEM
, PAGE_KERNEL
, 0,
227 node
, __builtin_return_address(0));
229 * Make sure we don't trigger recursive page faults in the
232 wrapper_vmalloc_sync_mappings();
238 void *lttng_kvzalloc_node(unsigned long size
, gfp_t flags
, int node
)
240 return lttng_kvmalloc_node(size
, flags
| __GFP_ZERO
, node
);
244 void *lttng_kvmalloc(unsigned long size
, gfp_t flags
)
246 return lttng_kvmalloc_node(size
, flags
, NUMA_NO_NODE
);
250 void *lttng_kvzalloc(unsigned long size
, gfp_t flags
)
252 return lttng_kvzalloc_node(size
, flags
, NUMA_NO_NODE
);
256 void lttng_kvfree(const void *addr
)
258 if (is_vmalloc_addr(addr
)) {
266 #endif /* _LTTNG_WRAPPER_VMALLOC_H */