[lttng-ust.git] / include / ust / processor.h

/* Copyright (C) 2009  Pierre-Marc Fournier
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 */

#ifndef UST_PROCESSOR_H
#define UST_PROCESSOR_H

#include <stddef.h>
#include <string.h>
#include <urcu-bp.h>

extern __thread long ust_reg_stack[500];
extern volatile __thread long *ust_reg_stack_ptr;

#define ____cacheline_aligned __attribute__((aligned(CAA_CACHE_LINE_SIZE)))

#ifdef __i386

struct registers {
	short ss;
	short cs;
	long esi;
	long ebp;
	long edx;
	long edi;
	long ecx;
	long ebx;
	long eax;
	long eflags;
	long esp;
};

static inline int fls(int x)
{
        int r;
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
        return r + 1;
}

#ifdef CONFIG_UST_GDB_INTEGRATION

/* save_registers - saves most of the processor's registers so
 * they are available to the probe. gdb uses this to give the
 * value of local variables.
 *
 * Saving all registers without losing any of their values is
 * tricky.
 *
 * We cannot pass to the asm stub the address of a registers structure
 * on the stack, because it will use a register and override its value.
 *
 * We don't want to use a stub to push the regs on the stack and then
 * another stub to copy them to a structure because changing %sp in asm
 * and then returning to C (even briefly) can have unexpected results.
 * Also, gcc might modify %sp between the stubs in reaction to the
 * register needs of the second stub that needs to know where to copy
 * the register values.
 *
 * So the chosen approach is to use another stack, declared in thread-
 * local storage, to push the registers. They are subsequently copied
 * to the stack, by C code.
 */

#define save_registers(regsptr) \
	asm volatile ( \
	     /* save original esp */ \
	     "pushl %%esp\n\t" \
	     /* push original eflags */ \
	     "pushfl\n\t" \
	      /* eax will hold the ptr to the private stack bottom */ \
	     "pushl %%eax\n\t" \
	     /* ebx is used for TLS access */ \
	     "pushl %%ebx\n\t" \
	     /* ecx will be used to temporarily hold the stack bottom addr */\
	     "pushl %%ecx\n\t"						     \
	     /* rdi is the input to __tls_get_addr, and also a temp var */   \
	     "pushl %%edi\n\t"						     \
	     /* For TLS access, we have to do function calls. However,	     \
	      * we must not lose the original value of:			     \
	      *   esp, eflags, eax, ebx, ecx, edx, esi, edi, ebp, cs, ss     \
	      *								     \
	      * Some registers' original values have already been saved:     \
	      *   esp, eflags, eax, ebx, ecx, edi			     \
	      *								     \
	      * In addition, the i386 ABI says the following registers belong\
	      * to the caller function:					     \
	      *   esp, ebp, esi, edi, ebx				     \
	      *								     \
	      * The following registers should not be changed by the callee: \
	      *   cs, ss						     \
	      *								     \
	      * Therefore, the following registers must be explicitly 	     \
	      * preserved:						     \
	      *   edx							     \
	      */ \
	     "pushl %%edx\n\t" \
	     /* Get GOT address */ \
	     "call __i686.get_pc_thunk.bx\n\t" \
	     "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" \
	     /* Start TLS access of private reg stack pointer */ \
	     "leal ust_reg_stack_ptr@tlsgd(,%%ebx,1),%%eax\n\t" \
	     "call ___tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     /* check if ust_reg_stack_ptr has been initialized */ \
	     "movl (%%eax),%%ecx\n\t" \
	     "testl %%ecx,%%ecx\n\t" \
	     "jne 1f\n\t" \
	     "movl %%eax,%%ecx\n\t" \
	     /* Save ecx because we are using it. */ \
	     "pushl %%ecx\n\t" \
	     /* Start TLS access of private reg stack */ \
	     "leal ust_reg_stack@tlsgd(,%%ebx,1),%%eax\n\t" \
	     "call ___tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     "popl %%ecx\n\t" \
	     "addl $500,%%eax\n\t" \
	     "movl %%eax,(%%ecx)\n\t" \
	     "movl %%ecx,%%eax\n\t" \
	     /* now the pointer to the private stack is in eax. \
	        must add stack size so the ptr points to the stack bottom. */ \
	"1:\n\t" \
	     /* edx was pushed for function calls */ \
	     "popl %%edx\n\t" \
	     /* Manually push esp to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 20(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push eflags to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 16(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push eax to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 12(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push ebx to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 8(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push ecx to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 4(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push edi to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 0(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* now push regs to tls */ \
	     /* -- esp already pushed -- */ \
	     /* -- eax already pushed -- */ \
	     /* -- ebx already pushed -- */ \
	     /* -- ecx already pushed -- */ \
	     /* -- edi already pushed -- */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edx,(%%ebx)\n\t" \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%ebp,(%%ebx)\n\t" \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%esi,(%%ebx)\n\t" \
	     /* push cs */ \
	     "addl $-2,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movw %%cs, (%%ebx)\n\t" \
	     /* push ss */ \
	     "addl $-2,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movw %%ss, (%%ebx)\n\t" \
	     /* restore original values of regs that were used internally */ \
	     "popl %%edi\n\t" \
	     "popl %%ecx\n\t" \
	     "popl %%ebx\n\t" \
	     "popl %%eax\n\t" \
	     /* cancel push of rsp */ \
	     "addl $4,%%esp\n\t" \
	     /* cancel push of eflags */ \
	     "addl $4,%%esp\n\t" \
	     ::: "memory"); \
	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));

#else /* CONFIG_UST_GDB_INTEGRATION */

#define save_registers(a)

#endif /* CONFIG_UST_GDB_INTEGRATION */

#define RELATIVE_ADDRESS(__rel_label__) __rel_label__

#define ARCH_COPY_ADDR(dst) "lea 2b," dst "\n\t"

#define _ASM_PTR ".long "

#elif defined(__x86_64)

struct registers {
	int padding; /* 4 bytes */
	short ss;
	short cs;
	unsigned long r15;
	unsigned long r14;
	unsigned long r13;
	unsigned long r12;
	unsigned long r11;
	unsigned long r10;
	unsigned long r9;
	unsigned long r8;
	unsigned long rsi;
	unsigned long rbp;
	unsigned long rdx;
	unsigned long rcx;
	unsigned long rdi;
	unsigned long rbx;
	unsigned long rax;
	unsigned long rflags;
	unsigned long rsp;
};

static inline int fls(int x)
{
        int r;
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
        return r + 1;
}

#ifdef CONFIG_UST_GDB_INTEGRATION

#define save_registers(regsptr) \
	asm volatile ( \
	     /* save original rsp */ \
	     "pushq %%rsp\n\t" \
	     /* push original rflags */ \
	     "pushfq\n\t" \
	      /* rax will hold the ptr to the private stack bottom */ \
	     "pushq %%rax\n\t" \
	     /* rbx will be used to temporarily hold the stack bottom addr */ \
	     "pushq %%rbx\n\t" \
	     /* rdi is the input to __tls_get_addr, and also a temp var */ \
	     "pushq %%rdi\n\t" \
	     /* For TLS access, we have to do function calls. However,	     \
	      * we must not lose the original value of:			     \
	      *   rsp, rflags, rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9     \
	      *   r10, r11, r12, r13, r14, r15, cs, ss			     \
	      *								     \
	      * Some registers' original values have already been saved:     \
	      *   rsp, rflags, rax, rbx, rdi				     \
	      *								     \
	      * In addition, the x86-64 ABI says the following registers     \
	      * belong to the caller function:				     \
	      *   rbp, rbx, r12, r13, r14, r15				     \
	      *								     \
	      * The following registers should not be changed by the callee: \
	      *   cs, ss						     \
	      *								     \
	      * Therefore, the following registers must be explicitly 	     \
	      * preserved:						     \
	      *   rcx, rdx, rsi, r8, r9, r10, r11			     \
	      */ \
	     "pushq %%rcx\n\t" \
	     "pushq %%rdx\n\t" \
	     "pushq %%rsi\n\t" \
	     "pushq %%r8\n\t" \
	     "pushq %%r9\n\t" \
	     "pushq %%r10\n\t" \
	     "pushq %%r11\n\t" \
	     /* Start TLS access of private reg stack pointer */ \
	     ".byte 0x66\n\t" \
	     "leaq ust_reg_stack_ptr@tlsgd(%%rip), %%rdi\n\t" \
	     ".hword 0x6666\n\t" \
	     "rex64\n\t" \
	     "call __tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     /* check if ust_reg_stack_ptr has been initialized */ \
	     "movq (%%rax),%%rbx\n\t" \
	     "testq %%rbx,%%rbx\n\t" \
	     "jne 1f\n\t" \
	     "movq %%rax,%%rbx\n\t" \
	     /* Start TLS access of private reg stack */ \
	     ".byte 0x66\n\t" \
	     "leaq ust_reg_stack@tlsgd(%%rip), %%rdi\n\t" \
	     ".hword 0x6666\n\t" \
	     "rex64\n\t" \
	     "call __tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     "addq $500,%%rax\n\t" \
	     "movq %%rax,(%%rbx)\n\t" \
	     "movq %%rbx,%%rax\n\t" \
	     /* now the pointer to the private stack is in rax.
	        must add stack size so the ptr points to the stack bottom. */ \
	"1:\n\t" \
	     /* Pop regs that were pushed for function calls */ \
	     "popq %%r11\n\t" \
	     "popq %%r10\n\t" \
	     "popq %%r9\n\t" \
	     "popq %%r8\n\t" \
	     "popq %%rsi\n\t" \
	     "popq %%rdx\n\t" \
	     "popq %%rcx\n\t" \
	     /* Manually push rsp to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 32(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push eflags to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 24(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rax to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 16(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rbx to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 8(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rdi to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 0(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* now push regs to tls */ \
	     /* -- rsp already pushed -- */ \
	     /* -- rax already pushed -- */ \
	     /* -- rbx already pushed -- */ \
	     /* -- rdi already pushed -- */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rcx,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdx,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rbp,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rsi,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r8,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r9,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r10,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r11,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r12,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r13,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r14,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r15,(%%rbx)\n\t" \
	     /* push cs */ \
	     "addq $-2,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movw %%cs, (%%rbx)\n\t" \
	     /* push ss */ \
	     "addq $-2,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movw %%ss, (%%rbx)\n\t" \
	     /* add padding for struct registers */ \
	     "addq $-4,(%%rax)\n\t" \
	     /* restore original values of regs that were used internally */ \
	     "popq %%rdi\n\t" \
	     "popq %%rbx\n\t" \
	     "popq %%rax\n\t" \
	     /* cancel push of rsp */ \
	     "addq $8,%%rsp\n\t" \
	     /* cancel push of rflags */ \
	     "addq $8,%%rsp\n\t" \
	     ::); \
	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));

#else /* CONFIG_UST_GDB_INTEGRATION */

#define save_registers(a)

#endif /* CONFIG_UST_GDB_INTEGRATION */

/* Macro to insert the address of a relative jump in an assembly stub,
 * in a relocatable way. On x86-64, this uses a special (%rip) notation. */
#define RELATIVE_ADDRESS(__rel_label__) __rel_label__(%%rip)

#define ARCH_COPY_ADDR(dst) "lea 2b(%%rip)," dst "\n\t"

#define _ASM_PTR ".quad "

#elif defined(__PPC__)

struct registers {
};

static __inline__ int fls(unsigned int x)
{
        int lz;

        asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
        return 32 - lz;
}

#define ARCH_COPY_ADDR(dst) \
		     "lis " dst ",2b@h\n\t" /* load high bytes */ \
		     "ori " dst "," dst ",2b@l\n\t" /* load low bytes */

#define _ASM_PTR ".long "
#define save_registers(a)

#else /* arch-agnostic */

static __inline__ int fls(unsigned int x)
{
	int r = 32;

	if (!x)
		return 0;
	if (!(x & 0xFFFF0000U)) {
		x <<= 16;
		r -= 16;
	}
	if (!(x & 0xFF000000U)) {
		x <<= 8;
		r -= 8;
	}
	if (!(x & 0xF0000000U)) {
		x <<= 4;
		r -= 4;
	}
	if (!(x & 0xC0000000U)) {
		x <<= 2;
		r -= 2;
	}
	if (!(x & 0x80000000U)) {
		x <<= 1;
		r -= 1;
	}
	return r;
}

#endif

#endif /* UST_PROCESSOR_H */
Commit	Line	Data
a09dac63 PMF	1	/* Copyright (C) 2009 Pierre-Marc Fournier
	2	*
	3	* This library is free software; you can redistribute it and/or
	4	* modify it under the terms of the GNU Lesser General Public
	5	* License as published by the Free Software Foundation; either
	6	* version 2.1 of the License, or (at your option) any later version.
	7	*
	8	* This library is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	11	* Lesser General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU Lesser General Public
	14	* License along with this library; if not, write to the Free Software
	15	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	16	*/
	17
5af57e62 PMF	18	#ifndef UST_PROCESSOR_H
5af57e62 PMF	19	#define UST_PROCESSOR_H
d98a01c6 PMF	20
d98a01c6 PMF	21	#include <stddef.h>
636ca5d6	22	#include <string.h>
066b83b8	23	#include <urcu-bp.h>
636ca5d6 PMF	24
	25	extern __thread long ust_reg_stack[500];
	26	extern volatile __thread long *ust_reg_stack_ptr;
	27
0222e121	28	#define ____cacheline_aligned __attribute__((aligned(CAA_CACHE_LINE_SIZE)))
066b83b8	29
6fa0d97f	30	#ifdef __i386
d98a01c6 PMF	31
d98a01c6 PMF	32	struct registers {
7756d65a PMF	33	short ss;
7756d65a PMF	34	short cs;
d98a01c6	35	long esi;
7756d65a PMF	36	long ebp;
7756d65a PMF	37	long edx;
d98a01c6	38	long edi;
f2496f58	39	long ecx;
7756d65a PMF	40	long ebx;
7756d65a PMF	41	long eax;
d98a01c6	42	long eflags;
7756d65a	43	long esp;
d98a01c6 PMF	44	};
d98a01c6 PMF	45
6fa0d97f PMF	46	static inline int fls(int x)
	47	{
	48	int r;
	49	asm("bsrl %1,%0\n\t"
	50	"cmovzl %2,%0"
	51	: "=&r" (r) : "rm" (x), "rm" (-1));
	52	return r + 1;
	53	}
	54
e003d6ee	55	#ifdef CONFIG_UST_GDB_INTEGRATION
defa46a7	56
55c5b393 PMF	57	/* save_registers - saves most of the processor's registers so
	58	* they are available to the probe. gdb uses this to give the
	59	* value of local variables.
	60	*
	61	* Saving all registers without losing any of their values is
	62	* tricky.
	63	*
	64	* We cannot pass to the asm stub the address of a registers structure
	65	* on the stack, because it will use a register and override its value.
	66	*
	67	* We don't want to use a stub to push the regs on the stack and then
	68	* another stub to copy them to a structure because changing %sp in asm
	69	* and then returning to C (even briefly) can have unexpected results.
	70	* Also, gcc might modify %sp between the stubs in reaction to the
	71	* register needs of the second stub that needs to know where to copy
	72	* the register values.
	73	*
	74	* So the chosen approach is to use another stack, declared in thread-
	75	* local storage, to push the registers. They are subsequently copied
	76	* to the stack, by C code.
	77	*/
7756d65a PMF	78
	79	#define save_registers(regsptr) \
	80	asm volatile ( \
	81	/* save original esp */ \
	82	"pushl %%esp\n\t" \
	83	/* push original eflags */ \
	84	"pushfl\n\t" \
	85	/* eax will hold the ptr to the private stack bottom */ \
	86	"pushl %%eax\n\t" \
f2496f58	87	/* ebx is used for TLS access */ \
7756d65a	88	"pushl %%ebx\n\t" \
55c5b393 PMF	89	/* ecx will be used to temporarily hold the stack bottom addr */\
	90	"pushl %%ecx\n\t" \
	91	/* rdi is the input to __tls_get_addr, and also a temp var */ \
	92	"pushl %%edi\n\t" \
	93	/* For TLS access, we have to do function calls. However, \
	94	* we must not lose the original value of: \
	95	* esp, eflags, eax, ebx, ecx, edx, esi, edi, ebp, cs, ss \
	96	* \
	97	* Some registers' original values have already been saved: \
	98	* esp, eflags, eax, ebx, ecx, edi \
	99	* \
	100	* In addition, the i386 ABI says the following registers belong\
	101	* to the caller function: \
	102	* esp, ebp, esi, edi, ebx \
	103	* \
	104	* The following registers should not be changed by the callee: \
	105	* cs, ss \
	106	* \
	107	* Therefore, the following registers must be explicitly \
	108	* preserved: \
	109	* edx \
	110	*/ \
	111	"pushl %%edx\n\t" \
f2496f58 PMF	112	/* Get GOT address */ \
	113	"call __i686.get_pc_thunk.bx\n\t" \
	114	"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" \
7756d65a PMF	115	/* Start TLS access of private reg stack pointer */ \
	116	"leal ust_reg_stack_ptr@tlsgd(,%%ebx,1),%%eax\n\t" \
	117	"call ___tls_get_addr@plt\n\t" \
	118	/* --- End TLS access */ \
	119	/* check if ust_reg_stack_ptr has been initialized */ \
f2496f58 PMF	120	"movl (%%eax),%%ecx\n\t" \
f2496f58 PMF	121	"testl %%ecx,%%ecx\n\t" \
7756d65a	122	"jne 1f\n\t" \
f2496f58	123	"movl %%eax,%%ecx\n\t" \
55c5b393	124	/* Save ecx because we are using it. */ \
f2496f58	125	"pushl %%ecx\n\t" \
7756d65a PMF	126	/* Start TLS access of private reg stack */ \
	127	"leal ust_reg_stack@tlsgd(,%%ebx,1),%%eax\n\t" \
	128	"call ___tls_get_addr@plt\n\t" \
	129	/* --- End TLS access */ \
f2496f58	130	"popl %%ecx\n\t" \
7756d65a	131	"addl $500,%%eax\n\t" \
f2496f58 PMF	132	"movl %%eax,(%%ecx)\n\t" \
f2496f58 PMF	133	"movl %%ecx,%%eax\n\t" \
7756d65a PMF	134	/* now the pointer to the private stack is in eax. \
	135	must add stack size so the ptr points to the stack bottom. */ \
	136	"1:\n\t" \
55c5b393 PMF	137	/* edx was pushed for function calls */ \
55c5b393 PMF	138	"popl %%edx\n\t" \
7756d65a PMF	139	/* Manually push esp to private stack */ \
7756d65a PMF	140	"addl $-4,(%%eax)\n\t" \
f2496f58	141	"movl 20(%%esp), %%edi\n\t" \
7756d65a PMF	142	"movl (%%eax), %%ebx\n\t" \
	143	"movl %%edi, (%%ebx)\n\t" \
	144	/* Manually push eflags to private stack */ \
	145	"addl $-4,(%%eax)\n\t" \
f2496f58	146	"movl 16(%%esp), %%edi\n\t" \
7756d65a PMF	147	"movl (%%eax), %%ebx\n\t" \
	148	"movl %%edi, (%%ebx)\n\t" \
	149	/* Manually push eax to private stack */ \
	150	"addl $-4,(%%eax)\n\t" \
f2496f58	151	"movl 12(%%esp), %%edi\n\t" \
7756d65a PMF	152	"movl (%%eax), %%ebx\n\t" \
	153	"movl %%edi, (%%ebx)\n\t" \
	154	/* Manually push ebx to private stack */ \
	155	"addl $-4,(%%eax)\n\t" \
f2496f58 PMF	156	"movl 8(%%esp), %%edi\n\t" \
	157	"movl (%%eax), %%ebx\n\t" \
	158	"movl %%edi, (%%ebx)\n\t" \
	159	/* Manually push ecx to private stack */ \
	160	"addl $-4,(%%eax)\n\t" \
7756d65a PMF	161	"movl 4(%%esp), %%edi\n\t" \
	162	"movl (%%eax), %%ebx\n\t" \
	163	"movl %%edi, (%%ebx)\n\t" \
	164	/* Manually push edi to private stack */ \
	165	"addl $-4,(%%eax)\n\t" \
	166	"movl 0(%%esp), %%edi\n\t" \
	167	"movl (%%eax), %%ebx\n\t" \
	168	"movl %%edi, (%%ebx)\n\t" \
	169	/* now push regs to tls */ \
	170	/* -- esp already pushed -- */ \
	171	/* -- eax already pushed -- */ \
	172	/* -- ebx already pushed -- */ \
f2496f58	173	/* -- ecx already pushed -- */ \
7756d65a PMF	174	/* -- edi already pushed -- */ \
	175	"addl $-4,(%%eax)\n\t" \
	176	"movl (%%eax), %%ebx\n\t" \
7756d65a PMF	177	"movl %%edx,(%%ebx)\n\t" \
	178	"addl $-4,(%%eax)\n\t" \
	179	"movl (%%eax), %%ebx\n\t" \
	180	"movl %%ebp,(%%ebx)\n\t" \
	181	"addl $-4,(%%eax)\n\t" \
	182	"movl (%%eax), %%ebx\n\t" \
	183	"movl %%esi,(%%ebx)\n\t" \
	184	/* push cs */ \
	185	"addl $-2,(%%eax)\n\t" \
	186	"movl (%%eax), %%ebx\n\t" \
	187	"movw %%cs, (%%ebx)\n\t" \
	188	/* push ss */ \
	189	"addl $-2,(%%eax)\n\t" \
	190	"movl (%%eax), %%ebx\n\t" \
	191	"movw %%ss, (%%ebx)\n\t" \
	192	/* restore original values of regs that were used internally */ \
	193	"popl %%edi\n\t" \
f2496f58	194	"popl %%ecx\n\t" \
7756d65a PMF	195	"popl %%ebx\n\t" \
	196	"popl %%eax\n\t" \
	197	/* cancel push of rsp */ \
	198	"addl $4,%%esp\n\t" \
	199	/* cancel push of eflags */ \
	200	"addl $4,%%esp\n\t" \
	201	::: "memory"); \
	202	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	203	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));
defa46a7	204
e003d6ee	205	#else /* CONFIG_UST_GDB_INTEGRATION */
defa46a7	206
defa46a7 PMF	207	#define save_registers(a)
defa46a7 PMF	208
e003d6ee	209	#endif /* CONFIG_UST_GDB_INTEGRATION */
d98a01c6	210
9e8f4f52 PMF	211	#define RELATIVE_ADDRESS(__rel_label__) __rel_label__
9e8f4f52 PMF	212
6fa0d97f	213	#define ARCH_COPY_ADDR(dst) "lea 2b," dst "\n\t"
9692fa18	214
9e8f4f52 PMF	215	#define _ASM_PTR ".long "
9e8f4f52 PMF	216
7f0357f0	217	#elif defined(__x86_64)
d98a01c6 PMF	218
d98a01c6 PMF	219	struct registers {
636ca5d6 PMF	220	int padding; /* 4 bytes */
	221	short ss;
	222	short cs;
636ca5d6 PMF	223	unsigned long r15;
	224	unsigned long r14;
	225	unsigned long r13;
	226	unsigned long r12;
	227	unsigned long r11;
	228	unsigned long r10;
	229	unsigned long r9;
	230	unsigned long r8;
d98a01c6	231	unsigned long rsi;
636ca5d6 PMF	232	unsigned long rbp;
	233	unsigned long rdx;
	234	unsigned long rcx;
d98a01c6	235	unsigned long rdi;
636ca5d6 PMF	236	unsigned long rbx;
636ca5d6 PMF	237	unsigned long rax;
a5850bc4	238	unsigned long rflags;
636ca5d6	239	unsigned long rsp;
d98a01c6 PMF	240	};
d98a01c6 PMF	241
6fa0d97f PMF	242	static inline int fls(int x)
	243	{
	244	int r;
	245	asm("bsrl %1,%0\n\t"
	246	"cmovzl %2,%0"
	247	: "=&r" (r) : "rm" (x), "rm" (-1));
	248	return r + 1;
	249	}
	250
e003d6ee	251	#ifdef CONFIG_UST_GDB_INTEGRATION
defa46a7	252
8524c98d	253	#define save_registers(regsptr) \
636ca5d6 PMF	254	asm volatile ( \
	255	/* save original rsp */ \
	256	"pushq %%rsp\n\t" \
a5850bc4 PMF	257	/* push original rflags */ \
a5850bc4 PMF	258	"pushfq\n\t" \
636ca5d6 PMF	259	/* rax will hold the ptr to the private stack bottom */ \
	260	"pushq %%rax\n\t" \
	261	/* rbx will be used to temporarily hold the stack bottom addr */ \
	262	"pushq %%rbx\n\t" \
	263	/* rdi is the input to __tls_get_addr, and also a temp var */ \
	264	"pushq %%rdi\n\t" \
55c5b393 PMF	265	/* For TLS access, we have to do function calls. However, \
	266	* we must not lose the original value of: \
	267	* rsp, rflags, rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9 \
	268	* r10, r11, r12, r13, r14, r15, cs, ss \
	269	* \
	270	* Some registers' original values have already been saved: \
	271	* rsp, rflags, rax, rbx, rdi \
	272	* \
	273	* In addition, the x86-64 ABI says the following registers \
	274	* belong to the caller function: \
	275	* rbp, rbx, r12, r13, r14, r15 \
	276	* \
	277	* The following registers should not be changed by the callee: \
	278	* cs, ss \
	279	* \
	280	* Therefore, the following registers must be explicitly \
	281	* preserved: \
	282	* rcx, rdx, rsi, r8, r9, r10, r11 \
	283	*/ \
	284	"pushq %%rcx\n\t" \
	285	"pushq %%rdx\n\t" \
	286	"pushq %%rsi\n\t" \
	287	"pushq %%r8\n\t" \
	288	"pushq %%r9\n\t" \
	289	"pushq %%r10\n\t" \
	290	"pushq %%r11\n\t" \
a5850bc4	291	/* Start TLS access of private reg stack pointer */ \
636ca5d6 PMF	292	".byte 0x66\n\t" \
636ca5d6 PMF	293	"leaq ust_reg_stack_ptr@tlsgd(%%rip), %%rdi\n\t" \
34174843	294	".hword 0x6666\n\t" \
636ca5d6 PMF	295	"rex64\n\t" \
	296	"call __tls_get_addr@plt\n\t" \
	297	/* --- End TLS access */ \
a5850bc4 PMF	298	/* check if ust_reg_stack_ptr has been initialized */ \
	299	"movq (%%rax),%%rbx\n\t" \
	300	"testq %%rbx,%%rbx\n\t" \
	301	"jne 1f\n\t" \
	302	"movq %%rax,%%rbx\n\t" \
	303	/* Start TLS access of private reg stack */ \
	304	".byte 0x66\n\t" \
	305	"leaq ust_reg_stack@tlsgd(%%rip), %%rdi\n\t" \
34174843	306	".hword 0x6666\n\t" \
a5850bc4 PMF	307	"rex64\n\t" \
	308	"call __tls_get_addr@plt\n\t" \
	309	/* --- End TLS access */ \
	310	"addq $500,%%rax\n\t" \
	311	"movq %%rax,(%%rbx)\n\t" \
	312	"movq %%rbx,%%rax\n\t" \
	313	/* now the pointer to the private stack is in rax.
	314	must add stack size so the ptr points to the stack bottom. */ \
	315	"1:\n\t" \
55c5b393 PMF	316	/* Pop regs that were pushed for function calls */ \
	317	"popq %%r11\n\t" \
	318	"popq %%r10\n\t" \
	319	"popq %%r9\n\t" \
	320	"popq %%r8\n\t" \
	321	"popq %%rsi\n\t" \
	322	"popq %%rdx\n\t" \
	323	"popq %%rcx\n\t" \
636ca5d6 PMF	324	/* Manually push rsp to private stack */ \
636ca5d6 PMF	325	"addq $-8,(%%rax)\n\t" \
a5850bc4 PMF	326	"movq 32(%%rsp), %%rdi\n\t" \
	327	"movq (%%rax), %%rbx\n\t" \
	328	"movq %%rdi, (%%rbx)\n\t" \
	329	/* Manually push eflags to private stack */ \
	330	"addq $-8,(%%rax)\n\t" \
636ca5d6 PMF	331	"movq 24(%%rsp), %%rdi\n\t" \
	332	"movq (%%rax), %%rbx\n\t" \
	333	"movq %%rdi, (%%rbx)\n\t" \
	334	/* Manually push rax to private stack */ \
	335	"addq $-8,(%%rax)\n\t" \
	336	"movq 16(%%rsp), %%rdi\n\t" \
	337	"movq (%%rax), %%rbx\n\t" \
	338	"movq %%rdi, (%%rbx)\n\t" \
	339	/* Manually push rbx to private stack */ \
	340	"addq $-8,(%%rax)\n\t" \
	341	"movq 8(%%rsp), %%rdi\n\t" \
	342	"movq (%%rax), %%rbx\n\t" \
	343	"movq %%rdi, (%%rbx)\n\t" \
	344	/* Manually push rdi to private stack */ \
	345	"addq $-8,(%%rax)\n\t" \
	346	"movq 0(%%rsp), %%rdi\n\t" \
	347	"movq (%%rax), %%rbx\n\t" \
	348	"movq %%rdi, (%%rbx)\n\t" \
	349	/* now push regs to tls */ \
	350	/* -- rsp already pushed -- */ \
	351	/* -- rax already pushed -- */ \
	352	/* -- rbx already pushed -- */ \
	353	/* -- rdi already pushed -- */ \
	354	"addq $-8,(%%rax)\n\t" \
	355	"movq (%%rax), %%rbx\n\t" \
	356	"movq %%rcx,(%%rbx)\n\t" \
	357	"addq $-8,(%%rax)\n\t" \
	358	"movq (%%rax), %%rbx\n\t" \
	359	"movq %%rdx,(%%rbx)\n\t" \
	360	"addq $-8,(%%rax)\n\t" \
	361	"movq (%%rax), %%rbx\n\t" \
	362	"movq %%rbp,(%%rbx)\n\t" \
	363	"addq $-8,(%%rax)\n\t" \
	364	"movq (%%rax), %%rbx\n\t" \
	365	"movq %%rsi,(%%rbx)\n\t" \
	366	"addq $-8,(%%rax)\n\t" \
	367	"movq (%%rax), %%rbx\n\t" \
	368	"movq %%r8,(%%rbx)\n\t" \
	369	"addq $-8,(%%rax)\n\t" \
	370	"movq (%%rax), %%rbx\n\t" \
	371	"movq %%r9,(%%rbx)\n\t" \
	372	"addq $-8,(%%rax)\n\t" \
	373	"movq (%%rax), %%rbx\n\t" \
	374	"movq %%r10,(%%rbx)\n\t" \
	375	"addq $-8,(%%rax)\n\t" \
	376	"movq (%%rax), %%rbx\n\t" \
	377	"movq %%r11,(%%rbx)\n\t" \
	378	"addq $-8,(%%rax)\n\t" \
	379	"movq (%%rax), %%rbx\n\t" \
	380	"movq %%r12,(%%rbx)\n\t" \
	381	"addq $-8,(%%rax)\n\t" \
	382	"movq (%%rax), %%rbx\n\t" \
	383	"movq %%r13,(%%rbx)\n\t" \
	384	"addq $-8,(%%rax)\n\t" \
	385	"movq (%%rax), %%rbx\n\t" \
	386	"movq %%r14,(%%rbx)\n\t" \
	387	"addq $-8,(%%rax)\n\t" \
	388	"movq (%%rax), %%rbx\n\t" \
	389	"movq %%r15,(%%rbx)\n\t" \
636ca5d6 PMF	390	/* push cs */ \
	391	"addq $-2,(%%rax)\n\t" \
	392	"movq (%%rax), %%rbx\n\t" \
	393	"movw %%cs, (%%rbx)\n\t" \
	394	/* push ss */ \
	395	"addq $-2,(%%rax)\n\t" \
	396	"movq (%%rax), %%rbx\n\t" \
	397	"movw %%ss, (%%rbx)\n\t" \
	398	/* add padding for struct registers */ \
	399	"addq $-4,(%%rax)\n\t" \
	400	/* restore original values of regs that were used internally */ \
	401	"popq %%rdi\n\t" \
	402	"popq %%rbx\n\t" \
	403	"popq %%rax\n\t" \
	404	/* cancel push of rsp */ \
	405	"addq $8,%%rsp\n\t" \
a5850bc4 PMF	406	/* cancel push of rflags */ \
a5850bc4 PMF	407	"addq $8,%%rsp\n\t" \
636ca5d6 PMF	408	::); \
	409	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	410	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));
d98a01c6	411
fc1f31ab PMF	412	#else /* CONFIG_UST_GDB_INTEGRATION */
fc1f31ab PMF	413
fc1f31ab PMF	414	#define save_registers(a)
fc1f31ab PMF	415
e003d6ee	416	#endif /* CONFIG_UST_GDB_INTEGRATION */
defa46a7	417
9e8f4f52 PMF	418	/* Macro to insert the address of a relative jump in an assembly stub,
	419	* in a relocatable way. On x86-64, this uses a special (%rip) notation. */
	420	#define RELATIVE_ADDRESS(__rel_label__) __rel_label__(%%rip)
	421
6fa0d97f	422	#define ARCH_COPY_ADDR(dst) "lea 2b(%%rip)," dst "\n\t"
9692fa18	423
9e8f4f52 PMF	424	#define _ASM_PTR ".quad "
9e8f4f52 PMF	425
7f0357f0	426	#elif defined(__PPC__)
6fa0d97f PMF	427
	428	struct registers {
	429	};
	430
	431	static __inline__ int fls(unsigned int x)
	432	{
	433	int lz;
	434
	435	asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
	436	return 32 - lz;
	437	}
	438
	439	#define ARCH_COPY_ADDR(dst) \
	440	"lis " dst ",2b@h\n\t" /* load high bytes */ \
	441	"ori " dst "," dst ",2b@l\n\t" /* load low bytes */
	442
	443	#define _ASM_PTR ".long "
	444	#define save_registers(a)
	445
7f0357f0 MD	446	#else /* arch-agnostic */
	447
	448	static __inline__ int fls(unsigned int x)
	449	{
	450	int r = 32;
	451
	452	if (!x)
	453	return 0;
	454	if (!(x & 0xFFFF0000U)) {
	455	x <<= 16;
	456	r -= 16;
	457	}
	458	if (!(x & 0xFF000000U)) {
	459	x <<= 8;
	460	r -= 8;
	461	}
	462	if (!(x & 0xF0000000U)) {
	463	x <<= 4;
	464	r -= 4;
	465	}
	466	if (!(x & 0xC0000000U)) {
	467	x <<= 2;
	468	r -= 2;
	469	}
	470	if (!(x & 0x80000000U)) {
	471	x <<= 1;
	472	r -= 1;
	473	}
	474	return r;
	475	}
	476
	477	#endif
d98a01c6	478
5af57e62	479	#endif /* UST_PROCESSOR_H */