[PATCH] target: i386: Check float overflow about register stack

[chengang]

From: Chen Gang <address@hidden>

The fxam instruction also checks the register stack overflow, which can
be get by the following fstsw instruction. The related code is below, it
works well under real x86_64 hardware, but can not work under qemu-i386.

0006b63c <_CIsqrt>:
   6b63c:       55                      push   %ebp
   6b63d:       89 e5                   mov    %esp,%ebp
   6b63f:       83 ec 44                sub    $0x44,%esp
   6b642:       dd 1c 24                fstpl  (%esp)
   6b645:       9b                      fwait
   6b646:       e8 d5 04 00 00          call   6bb20 <wine_backtrace>
   6b64b:       b9 01 00 00 00          mov    $0x1,%ecx
   6b650:       d9 e5                   fxam
   6b652:       9b df e0                fstsw  %ax
   6b655:       66 25 00 45             and    $0x4500,%ax
   6b659:       66 3d 00 41             cmp    $0x4100,%ax
   6b65d:       74 07                   je     6b666 <_CIsqrt+0x2a>
   6b65f:       dd 1c cc                fstpl  (%esp,%ecx,8)
   6b662:       9b                      fwait
   6b663:       41                      inc    %ecx
   6b664:       eb ea                   jmp    6b650 <_CIsqrt+0x14>
   6b666:       89 4d fc                mov    %ecx,-0x4(%ebp)
   6b669:       e8 b2 0f 00 00          call   6c620 <MSVCRT_sqrt>
   6b66e:       8b 4d fc                mov    -0x4(%ebp),%ecx
   6b671:       dd 1c 24                fstpl  (%esp)
   6b674:       49                      dec    %ecx
   6b675:       dd 04 cc                fldl   (%esp,%ecx,8)
   6b678:       83 f9 00                cmp    $0x0,%ecx
   6b67b:       75 f7                   jne    6b674 <_CIsqrt+0x38>
   6b67d:       c9                      leave
   6b67e:       c3                      ret
   6b67f:       90                      nop

Signed-off-by: Chen Gang <address@hidden>
---
 target/i386/cpu.h        |  1 +
 target/i386/fpu_helper.c | 70 ++++++++++++++++++++++++----------------
 2 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 576f309bbf..3e2b719ab7 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1394,6 +1394,7 @@ typedef struct CPUX86State {
     struct {} start_init_save;
 
     /* FPU state */
+    bool foverflow;
     unsigned int fpstt; /* top of stack index */
     uint16_t fpus;
     uint16_t fpuc;
diff --git a/target/i386/fpu_helper.c b/target/i386/fpu_helper.c
index 99f28f267f..81f3cefe8b 100644
--- a/target/i386/fpu_helper.c
+++ b/target/i386/fpu_helper.c
@@ -91,17 +91,31 @@ void cpu_set_ignne(void)
 }
 #endif
 
+static inline void set_fpstt(CPUX86State *env, unsigned int fpstt,
+                             bool pop, bool full)
+{
+    env->foverflow = (fpstt > 7) && full; /* clear the original flag */
+    if (pop) {
+        if (full) {
+            env->fptags[env->fpstt] = 1; /* invalidate stack entry */
+        }
+        env->fpstt = fpstt & 7;
+    } else {
+        env->fpstt = fpstt & 7;
+        if (full) {
+            env->fptags[env->fpstt] = 0; /* validate stack entry */
+        }
+    }
+}
 
 static inline void fpush(CPUX86State *env)
 {
-    env->fpstt = (env->fpstt - 1) & 7;
-    env->fptags[env->fpstt] = 0; /* validate stack entry */
+    set_fpstt(env, env->fpstt - 1, false, true);
 }
 
 static inline void fpop(CPUX86State *env)
 {
-    env->fptags[env->fpstt] = 1; /* invalidate stack entry */
-    env->fpstt = (env->fpstt + 1) & 7;
+    set_fpstt(env, env->fpstt + 1, true, true);
 }
 
 static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
@@ -211,11 +225,10 @@ void helper_flds_ST0(CPUX86State *env, uint32_t val)
         uint32_t i;
     } u;
 
-    new_fpstt = (env->fpstt - 1) & 7;
+    new_fpstt = env->fpstt - 1;
     u.i = val;
-    env->fpregs[new_fpstt].d = float32_to_floatx80(u.f, &env->fp_status);
-    env->fpstt = new_fpstt;
-    env->fptags[new_fpstt] = 0; /* validate stack entry */
+    env->fpregs[new_fpstt & 7].d = float32_to_floatx80(u.f, &env->fp_status);
+    set_fpstt(env, new_fpstt, false, true);
 }
 
 void helper_fldl_ST0(CPUX86State *env, uint64_t val)
@@ -226,31 +239,28 @@ void helper_fldl_ST0(CPUX86State *env, uint64_t val)
         uint64_t i;
     } u;
 
-    new_fpstt = (env->fpstt - 1) & 7;
+    new_fpstt = env->fpstt - 1;
     u.i = val;
-    env->fpregs[new_fpstt].d = float64_to_floatx80(u.f, &env->fp_status);
-    env->fpstt = new_fpstt;
-    env->fptags[new_fpstt] = 0; /* validate stack entry */
+    env->fpregs[new_fpstt & 7].d = float64_to_floatx80(u.f, &env->fp_status);
+    set_fpstt(env, new_fpstt, false, true);
 }
 
 void helper_fildl_ST0(CPUX86State *env, int32_t val)
 {
     int new_fpstt;
 
-    new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = int32_to_floatx80(val, &env->fp_status);
-    env->fpstt = new_fpstt;
-    env->fptags[new_fpstt] = 0; /* validate stack entry */
+    new_fpstt = env->fpstt - 1;
+    env->fpregs[new_fpstt & 7].d = int32_to_floatx80(val, &env->fp_status);
+    set_fpstt(env, new_fpstt, false, true);
 }
 
 void helper_fildll_ST0(CPUX86State *env, int64_t val)
 {
     int new_fpstt;
 
-    new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = int64_to_floatx80(val, &env->fp_status);
-    env->fpstt = new_fpstt;
-    env->fptags[new_fpstt] = 0; /* validate stack entry */
+    new_fpstt = env->fpstt - 1;
+    env->fpregs[new_fpstt & 7].d = int64_to_floatx80(val, &env->fp_status);
+    set_fpstt(env, new_fpstt, false, true);
 }
 
 uint32_t helper_fsts_ST0(CPUX86State *env)
@@ -345,10 +355,9 @@ void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
 {
     int new_fpstt;
 
-    new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = helper_fldt(env, ptr, GETPC());
-    env->fpstt = new_fpstt;
-    env->fptags[new_fpstt] = 0; /* validate stack entry */
+    new_fpstt = env->fpstt - 1;
+    env->fpregs[new_fpstt & 7].d = helper_fldt(env, ptr, GETPC());
+    set_fpstt(env, new_fpstt, false, true);
 }
 
 void helper_fstt_ST0(CPUX86State *env, target_ulong ptr)
@@ -368,13 +377,13 @@ void helper_fpop(CPUX86State *env)
 
 void helper_fdecstp(CPUX86State *env)
 {
-    env->fpstt = (env->fpstt - 1) & 7;
+    set_fpstt(env, env->fpstt - 1, false, false);
     env->fpus &= ~0x4700;
 }
 
 void helper_fincstp(CPUX86State *env)
 {
-    env->fpstt = (env->fpstt + 1) & 7;
+    set_fpstt(env, env->fpstt + 1, true, false);
     env->fpus &= ~0x4700;
 }
 
@@ -382,6 +391,7 @@ void helper_fincstp(CPUX86State *env)
 
 void helper_ffree_STN(CPUX86State *env, int st_index)
 {
+    set_fpstt(env, env->fpstt + st_index, true, false);
     env->fptags[(env->fpstt + st_index) & 7] = 1;
 }
 
@@ -644,6 +654,7 @@ void helper_fninit(CPUX86State *env)
 {
     env->fpus = 0;
     env->fpstt = 0;
+    env->foverflow = false;
     cpu_set_fpuc(env, 0x37f);
     env->fptags[0] = 1;
     env->fptags[1] = 1;
@@ -1008,6 +1019,11 @@ void helper_fxam_ST0(CPUX86State *env)
     } else {
         env->fpus |= 0x400;
     }
+
+    if (env->foverflow) {
+        env->fpus |= 0x4100;
+        env->fpus &= ~0x400;
+    }
 }
 
 static void do_fstenv(CPUX86State *env, target_ulong ptr, int data32,
@@ -1636,7 +1652,7 @@ void helper_ldmxcsr(CPUX86State *env, uint32_t val)
 
 void helper_enter_mmx(CPUX86State *env)
 {
-    env->fpstt = 0;
+    set_fpstt(env, 0, true, false);
     *(uint32_t *)(env->fptags) = 0;
     *(uint32_t *)(env->fptags + 4) = 0;
 }
-- 
2.24.0.308.g228f53135a