Commit 3ae23d5b authored by unknown's avatar unknown
Browse files

Bug #9854 hex() and out of range handling 

  added out of range handling
parent 73fd169d

mysql-test/r/func_str.result

+3 −0
Original line number Diff line number Diff line
@@ -865,3 +865,6 @@ abc abc c bc abc 
abcd	abcd	d	cd	bcd	abcd	

abcde	abcde	e	de	cde	bcde	abcde

drop table t1;

select hex(29223372036854775809), hex(-29223372036854775809);

hex(29223372036854775809)	hex(-29223372036854775809)

FFFFFFFFFFFFFFFF	FFFFFFFFFFFFFFFF

mysql-test/t/func_str.test

+5 −0
Original line number Diff line number Diff line
@@ -606,3 +606,8 @@ insert t1 values ('ab'), ('abc'), ('abcd'), ('abcde'); 
select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1;

select * from (select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1) t;

drop table t1;



#

# Bug #9854  hex() and out of range handling

#

select hex(29223372036854775809), hex(-29223372036854775809);

sql/item_strfunc.cc

+14 −2
Original line number Diff line number Diff line
@@ -2348,9 +2348,21 @@ String *Item_func_hex::val_str(String *str) 
  DBUG_ASSERT(fixed == 1);

  if (args[0]->result_type() != STRING_RESULT)

  {

    /* Return hex of unsigned longlong value */

    longlong dec= args[0]->val_int();

    ulonglong dec;

    char ans[65],*ptr;

    /* Return hex of unsigned longlong value */

    if (args[0]->result_type() == REAL_RESULT)

    {

      double val= args[0]->val();

      if ((val <= (double) LONGLONG_MIN) || 

          (val >= (double) (ulonglong) ULONGLONG_MAX))

        dec=  ~(longlong) 0;

      else

        dec= (ulonglong) (val + (val > 0 ? 0.5 : -0.5));

    }

    else

      dec= (ulonglong) args[0]->val_int();



    if ((null_value= args[0]->null_value))

      return 0;

    ptr= longlong2str(dec,ans,16);