Commit d0cf2fa6 authored by Chad MILLER's avatar Chad MILLER
Browse files

Bug#36270: incorrect calculation result - works in 4.1 but not in 5.0 or 5.1 

When the fractional part in a multiplication of DECIMALs
overflowed, we truncated the first operand rather than the
longest. Now truncating least significant places instead
for more precise multiplications.

(Queuing at demand of Trudy/Davi.)
parent 18e93e8f

mysql-test/r/type_newdecimal.result

+5 −0
Original line number Diff line number Diff line
@@ -1519,4 +1519,9 @@ SELECT f1 FROM t1; 
f1

99999999999999999999999999999.999999999999999999999999999999

DROP TABLE t1;

select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *

1.01500000 * 1.01500000 * 0.99500000);

(1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *

1.01500000 * 1.01500000 * 0.99500000)

0.812988073953673124592306939480

End of 5.0 tests

mysql-test/t/type_newdecimal.test

+9 −0
Original line number Diff line number Diff line
@@ -1216,4 +1216,13 @@ DESC t1; 
SELECT f1 FROM t1;

DROP TABLE t1;



#

# Bug #36270: incorrect calculation result - works in 4.1 but not in 5.0 or 5.1

#



# show that if we need to truncate the scale of an operand, we pick the

# right one (that is, we discard the least significant decimal places)

select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *

        1.01500000 * 1.01500000 * 0.99500000);



--echo End of 5.0 tests

strings/decimal.c

+15 −7
Original line number Diff line number Diff line
@@ -1999,18 +1999,18 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to) 


  sanity(to);



  i=intg0;

  i=intg0;                                       /* save 'ideal' values */

  j=frac0;

  FIX_INTG_FRAC_ERROR(to->len, intg0, frac0, error);

  FIX_INTG_FRAC_ERROR(to->len, intg0, frac0, error);  /* bound size */

  to->sign=from1->sign != from2->sign;

  to->frac=from1->frac+from2->frac;

  to->frac=from1->frac+from2->frac;              /* store size in digits */

  to->intg=intg0*DIG_PER_DEC1;



  if (unlikely(error))

  {

    set_if_smaller(to->frac, frac0*DIG_PER_DEC1);

    set_if_smaller(to->intg, intg0*DIG_PER_DEC1);

    if (unlikely(i > intg0))

    if (unlikely(i > intg0))                     /* bounded integer-part */

    {

      i-=intg0;

      j=i >> 1;
@@ -2018,13 +2018,21 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to) 
      intg2-=i-j;

      frac1=frac2=0; /* frac0 is already 0 here */

    }

    else

    else                                         /* bounded fract part */

    {

      j-=frac0;

      i=j >> 1;

      if (frac1 <= frac2)

      {

        frac1-= i;

        frac2-=j-i;

      }

      else

      {

        frac2-= i;

        frac1-=j-i;

      }

    }

  }

  start0=to->buf+intg0+frac0-1;

  start2=buf2+frac2-1;