target-mips: fix bad shifts in {dextp|dextpdp}

Fixed issues in the MIPSDSP64 instructions dextp and dextpdp.
Shifting can go out of 32 bit range.

https://bugs.launchpad.net/qemu/+bug/1631625

Reported-by: Thomas Huth <thuth@redhat.com>
Reported-by: Jia Liu <proljc@gmail.com>
Signed-off-by: Yongbok Kim <yongbok.kim@imgtec.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>

diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
index df7d220..dc70793 100644 (file)
--- a/target-mips/dsp_helper.c
+++ b/target-mips/dsp_helper.c
@@ -3477,7 +3477,7 @@ target_ulong helper_dextp(target_ulong ac, target_ulong size, CPUMIPSState *env)
 
     if (sub >= -1) {
         temp = (tempB << (64 - len)) | (tempA >> len);
-        temp = temp & ((0x01 << (size + 1)) - 1);
+        temp = temp & ((1ULL << (size + 1)) - 1);
         set_DSPControl_efi(0, env);
     } else {
         set_DSPControl_efi(1, env);
@@ -3506,7 +3506,7 @@ target_ulong helper_dextpdp(target_ulong ac, target_ulong size,
 
     if (sub >= -1) {
         temp = (tempB << (64 - len)) | (tempA >> len);
-        temp = temp & ((0x01 << (size + 1)) - 1);
+        temp = temp & ((1ULL << (size + 1)) - 1);
         set_DSPControl_pos(sub, env);
         set_DSPControl_efi(0, env);
     } else {