Attachment #84938 for bug #130028




#define set_fpu_swd(t,val) ((t)->thread.i387.fxsave.swd = (val))
#define set_fpu_fxsr_twd(t,val) ((t)->thread.i387.fxsave.twd = (val))

#define X87_FSW_ES (1 << 7)	/* Exception Summary */

/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
   is pending. Clear the x87 state here by setting it to fixed
   values. The kernel data segment can be sometimes 0 and sometimes
   new user value. Both should be ok.
   Use the PDA as safe address because it should be already in L1. */
static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
{
	if (unlikely(fx->swd & X87_FSW_ES))
		 asm volatile("fnclex");
	alternative_input(ASM_NOP8 ASM_NOP2,
	     	     "    emms\n"		/* clear stack tags */
	     	     "    fildl %%gs:0",	/* load to clear state */
		     X86_FEATURE_FXSAVE_LEAK);
}

static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) 
{ 
	int err;

#endif
	if (unlikely(err))
		__clear_user(fx, sizeof(struct i387_fxsave_struct));
	/* No need to clear here because the caller clears USED_MATH */
	return err;
} 


				"i" (offsetof(__typeof__(*tsk),
					      thread.i387.fxsave)));
#endif
	clear_fpu_state(&tsk->thread.i387.fxsave);
}

static inline void kernel_fpu_begin(void)

Lines 13-18 Link Here

(-)linux-2.6.16.8-work/include/asm-i386/i387.h (-4 / +26 lines)
13		13
14	#include <linux/sched.h>	14	#include <linux/sched.h>
15	#include <linux/init.h>	15	#include <linux/init.h>
		16	#include <linux/kernel_stat.h>
16	#include <asm/processor.h>	17	#include <asm/processor.h>
17	#include <asm/sigcontext.h>	18	#include <asm/sigcontext.h>
18	#include <asm/user.h>	19	#include <asm/user.h>
Lines 38-54 extern void init_fpu(struct task_struct Link Here
38	extern void kernel_fpu_begin(void);	39	extern void kernel_fpu_begin(void);
39	#define kernel_fpu_end() do { stts(); preempt_enable(); } while(0)	40	#define kernel_fpu_end() do { stts(); preempt_enable(); } while(0)
40		41
		42	/* We need a safe address that is cheap to find and that is already
		43	in L1 during context switch. The best choices are unfortunately
		44	different for UP and SMP */
		45	#ifdef CONFIG_SMP
		46	#define safe_address (__per_cpu_offset[0])
		47	#else
		48	#define safe_address (kstat_cpu(0).cpustat.user)
		49	#endif
		50
41	/*	51	/*
42	* These must be called with preempt disabled	52	* These must be called with preempt disabled
43	*/	53	*/
44	static inline void __save_init_fpu( struct task_struct *tsk )	54	static inline void __save_init_fpu( struct task_struct *tsk )
45	{	55	{
		56	/* Use more nops than strictly needed in case the compiler
		57	varies code */
46	alternative_input(	58	alternative_input(
47	"fnsave %1 ; fwait ;" GENERIC_NOP2,	59	"fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
48	"fxsave %1 ; fnclex",	60	"fxsave %[fx]\n"
		61	"bt $7,%[fsw] ; jc 1f ; fnclex\n1:",
49	X86_FEATURE_FXSR,	62	X86_FEATURE_FXSR,
50	"m" (tsk->thread.i387.fxsave)	63	[fx] "m" (tsk->thread.i387.fxsave),
51	:"memory");	64	[fsw] "m" (tsk->thread.i387.fxsave.swd) : "memory");
		65	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
		66	is pending. Clear the x87 state here by setting it to fixed
		67	values. __per_cpu_offset[0] is a random variable that should be in L1
		68	alternative_input(
		69	GENERIC_NOP8 GENERIC_NOP2,
		70	"emms\n\t" /* clear stack tags */
		71	"fildl %[addr]", /* set F?P to defined value */
		72	X86_FEATURE_FXSAVE_LEAK,
		73	[addr] "m" (safe_address));
52	task_thread_info(tsk)->status &= ~TS_USEDFPU;	74	task_thread_info(tsk)->status &= ~TS_USEDFPU;
53	}	75	}
54		76

Lines 207-212 static void __init init_amd(struct cpuin Link Here

(-)linux-2.6.16.8-work/arch/i386/kernel/cpu/amd.c (+2 lines)
207	set_bit(X86_FEATURE_K7, c->x86_capability);	207	set_bit(X86_FEATURE_K7, c->x86_capability);
208	break;	208	break;
209	}	209	}
		210	if (c->x86 >= 6)
		211	set_bit(X86_FEATURE_FXSAVE_LEAK, c->x86_capability);
210		212
211	display_cacheinfo(c);	213	display_cacheinfo(c);
212		214

Lines 909-914 static int __init init_amd(struct cpuinf Link Here

(-)linux-2.6.16.8-work/arch/x86_64/kernel/setup.c (+4 lines)
909	if (c->x86 == 15 && ((level >= 0x0f48 && level < 0x0f50) \|\| level >=	909	if (c->x86 == 15 && ((level >= 0x0f48 && level < 0x0f50) \|\| level >=
910	set_bit(X86_FEATURE_REP_GOOD, &c->x86_capability);	910	set_bit(X86_FEATURE_REP_GOOD, &c->x86_capability);
911		911
		912	/* Enable workaround for FXSAVE leak */
		913	if (c->x86 >= 6)
		914	set_bit(X86_FEATURE_FXSAVE_LEAK, &c->x86_capability);
		915
912	r = get_model_name(c);	916	r = get_model_name(c);
913	if (!r) {	917	if (!r) {
914	switch (c->x86) {	918	switch (c->x86) {

Lines 70-75 Link Here

(-)linux-2.6.16.8-work/include/asm-i386/cpufeature.h (+1 lines)
70	#define X86_FEATURE_P3 (332+ 6) / P3 */	70	#define X86_FEATURE_P3 (332+ 6) / P3 */
71	#define X86_FEATURE_P4 (332+ 7) / P4 */	71	#define X86_FEATURE_P4 (332+ 7) / P4 */
72	#define X86_FEATURE_CONSTANT_TSC (332+ 8) / TSC ticks at a constant rate */	72	#define X86_FEATURE_CONSTANT_TSC (332+ 8) / TSC ticks at a constant rate */
		73	#define X86_FEATURE_FXSAVE_LEAK (332+10) / FXSAVE leaks FOP/FIP/FOP */
73		74
74	/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */	75	/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
75	#define X86_FEATURE_XMM3 (432+ 0) / Streaming SIMD Extensions-3 */	76	#define X86_FEATURE_XMM3 (432+ 0) / Streaming SIMD Extensions-3 */

Lines 64-69 Link Here

(-)linux-2.6.16.8-work/include/asm-x86_64/cpufeature.h (+1 lines)
64	#define X86_FEATURE_REP_GOOD (332+ 4) / rep microcode works well on this	64	#define X86_FEATURE_REP_GOOD (332+ 4) / rep microcode works well on this
65	#define X86_FEATURE_CONSTANT_TSC (332+5) / TSC runs at constant rate */	65	#define X86_FEATURE_CONSTANT_TSC (332+5) / TSC runs at constant rate */
66	#define X86_FEATURE_SYNC_RDTSC (332+6) / RDTSC syncs CPU core */	66	#define X86_FEATURE_SYNC_RDTSC (332+6) / RDTSC syncs CPU core */
		67	#define X86_FEATURE_FXSAVE_LEAK (332+7) / FIP/FOP/FDP leaks through FXSAVE
67		68
68	/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */	69	/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
69	#define X86_FEATURE_XMM3 (432+ 0) / Streaming SIMD Extensions-3 */	70	#define X86_FEATURE_XMM3 (432+ 0) / Streaming SIMD Extensions-3 */

Lines 527-534 __switch_to(struct task_struct *prev_p, Link Here

(-)linux-2.6.16.8-work/arch/x86_64/kernel/process.c (-2 / +6 lines)
527	int cpu = smp_processor_id();	527	int cpu = smp_processor_id();
528	struct tss_struct *tss = &per_cpu(init_tss, cpu);	528	struct tss_struct *tss = &per_cpu(init_tss, cpu);
529		529
530	unlazy_fpu(prev_p);
531
532	/*	530	/*
533	* Reload esp0, LDT and the page table pointer:	531	* Reload esp0, LDT and the page table pointer:
534	*/	532	*/
Lines 591-596 __switch_to(struct task_struct *prev_p, Link Here
591	prev->userrsp = read_pda(oldrsp);	589	prev->userrsp = read_pda(oldrsp);
592	write_pda(oldrsp, next->userrsp);	590	write_pda(oldrsp, next->userrsp);
593	write_pda(pcurrent, next_p);	591	write_pda(pcurrent, next_p);
		592
		593	/* This must be here to ensure both math_state_restore() and
		594	kernel_fpu_begin() work consistently.
		595	And the AMD workaround requires it to be after DS reload. */
		596	unlazy_fpu(prev_p);
		597
594	write_pda(kernelstack,	598	write_pda(kernelstack,
595	task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);	599	task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);

Return to bug 130028

Lines 72-77 extern int set_fpregs(struct task_struct Link Here

(-)linux-2.6.16.8-work/include/asm-x86_64/i387.h (-1 / +19 lines)
72	#define set_fpu_swd(t,val) ((t)->thread.i387.fxsave.swd = (val))	72	#define set_fpu_swd(t,val) ((t)->thread.i387.fxsave.swd = (val))
73	#define set_fpu_fxsr_twd(t,val) ((t)->thread.i387.fxsave.twd = (val))	73	#define set_fpu_fxsr_twd(t,val) ((t)->thread.i387.fxsave.twd = (val))
74		74
		75	#define X87_FSW_ES (1 << 7) /* Exception Summary */
		76
		77	/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
		78	is pending. Clear the x87 state here by setting it to fixed
		79	values. The kernel data segment can be sometimes 0 and sometimes
		80	new user value. Both should be ok.
		81	Use the PDA as safe address because it should be already in L1. */
		82	static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
		83	{
		84	if (unlikely(fx->swd & X87_FSW_ES))
		85	asm volatile("fnclex");
		86	alternative_input(ASM_NOP8 ASM_NOP2,
		87	" emms\n" /* clear stack tags */
		88	" fildl %%gs:0", /* load to clear state */
		89	X86_FEATURE_FXSAVE_LEAK);
		90	}
		91
75	static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)	92	static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)
76	{	93	{
77	int err;	94	int err;
Lines 119-124 static inline int save_i387_checking(str Link Here
119	#endif	136	#endif
120	if (unlikely(err))	137	if (unlikely(err))
121	__clear_user(fx, sizeof(struct i387_fxsave_struct));	138	__clear_user(fx, sizeof(struct i387_fxsave_struct));
		139	/* No need to clear here because the caller clears USED_MATH */
122	return err;	140	return err;
123	}	141	}
124		142
Lines 149-155 static inline void __fxsave_clear(struct Link Here
149	"i" (offsetof(__typeof__(*tsk),	167	"i" (offsetof(__typeof__(*tsk),
150	thread.i387.fxsave)));	168	thread.i387.fxsave)));
151	#endif	169	#endif
152	__asm__ __volatile__("fnclex");	170	clear_fpu_state(&tsk->thread.i387.fxsave);
153	}	171	}
154		172
155	static inline void kernel_fpu_begin(void)	173	static inline void kernel_fpu_begin(void)