Commit e5ae30fd authored by sergefp@mysql.com's avatar sergefp@mysql.com
Browse files

BUG#19618: Crash for unsigned_col NOT IN (-1, ... ) 

- When manually constructing a SEL_TREE for "t.key NOT IN(...)", take into account that 
  get_mm_parts may return a tree with type SEL_TREE::IMPOSSIBLE
- Added missing OOM checks
- Added comments
parent 98b81c4f

mysql-test/r/func_in.result

+17 −0
Original line number Diff line number Diff line
@@ -326,3 +326,20 @@ deallocate prepare s; 
set @str=NULL;

drop table t2;

drop table t1;

create table t1 (

some_id smallint(5) unsigned,

key (some_id)

);

insert into t1 values (1),(2);

select some_id from t1 where some_id not in(2,-1);

some_id

1

select some_id from t1 where some_id not in(-4,-1,-4);

some_id

1

2

select some_id from t1 where some_id not in(-4,-1,3423534,2342342);

some_id

1

2

drop table t1;

mysql-test/t/func_in.test

+12 −0
Original line number Diff line number Diff line
@@ -220,3 +220,15 @@ set @str=NULL; 
drop table t2;

drop table t1;



# BUG#19618: Crash in range optimizer for 

#   "unsigned_keypart NOT IN(negative_number,...)" 

#   (introduced in fix BUG#15872) 

create table t1 (

  some_id smallint(5) unsigned,

  key (some_id)

);

insert into t1 values (1),(2);

select some_id from t1 where some_id not in(2,-1);

select some_id from t1 where some_id not in(-4,-1,-4);

select some_id from t1 where some_id not in(-4,-1,3423534,2342342);

drop table t1;

sql/opt_range.cc

+51 −18
Original line number Diff line number Diff line
@@ -3530,17 +3530,38 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func, 
        if (!value_item)

          break;

        

        /* Get a SEL_TREE for "-inf < X < c_0" interval */

        func->array->value_to_item(0, value_item);

        /* 

          Get a SEL_TREE for "(-inf|NULL) < X < c_0" interval.

          Note: for partially-covering keys the returned tree may represent

          a half-closed interval (-inf < X <= c_0). In that case the for the

          whole NOT IN statement the (-inf < X < +inf) interval will be

          constructed. It doesn't make sense to consider range access over

          such intervals, but we don't eliminate them here as 1) they are 

          handled correctly by all parts of the code, and 2) the case where

          such intervals are constructed is rare.

        */

        uint i=0;

        do 

        {

          func->array->value_to_item(i, value_item);

          tree= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC,

                             value_item, cmp_type);

          if (!tree)

            break;

          i++;

        } while (i < func->array->count && tree->type == SEL_TREE::IMPOSSIBLE);



        if (!tree || tree->type == SEL_TREE::IMPOSSIBLE)

        {

          /* We get here in cases like "t.unsigned NOT IN (-1,-2,-3) */

          tree= NULL;

          break;

        }

#define NOT_IN_IGNORE_THRESHOLD 1000        

        SEL_TREE *tree2;

        if (func->array->count < NOT_IN_IGNORE_THRESHOLD)

        {

          for (uint i=1; i < func->array->count; i++)

          for (; i < func->array->count; i++)

          {

            if (func->array->compare_elems(i, i-1))

            {
@@ -3548,18 +3569,27 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func, 
              func->array->value_to_item(i, value_item);

              tree2= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC,

                                  value_item, cmp_type);

              if (!tree2)

              {

                tree= NULL;

                break;

              }



              /* Change all intervals to be "c_{i-1} < X < c_i" */

              for (uint idx= 0; idx < param->keys; idx++)

              {

                SEL_ARG *new_interval;

                if ((new_interval=  tree2->keys[idx]))

                SEL_ARG *new_interval, *last_val;

                if (((new_interval= tree2->keys[idx])) && 

                    ((last_val= tree->keys[idx]->last())))

                {

                  SEL_ARG *last_val= tree->keys[idx]->last();

                  new_interval->min_value= last_val->max_value;

                  new_interval->min_flag= NEAR_MIN;

                }

              }

              /* 

                The following doesn't try to allocate memory so no need to

                check for NULL.

              */

              tree= tree_or(param, tree, tree2);

            }

          }
@@ -3567,6 +3597,8 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func, 
        else

          func->array->value_to_item(func->array->count - 1, value_item);

        

        if (tree && tree->type != SEL_TREE::IMPOSSIBLE)

        {

          /* 

            Get the SEL_TREE for the last "c_last < X < +inf" interval 

            (value_item cotains c_last already)
@@ -3575,6 +3607,7 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func, 
                              value_item, cmp_type);

          tree= tree_or(param, tree, tree2);

        }

      }

      else

      {

        tree= get_ne_mm_tree(param, cond_func, field,