Commit 0aabb89e authored by gshchepa/uchum@gleb.loc's avatar gshchepa/uchum@gleb.loc
Browse files

Fixed bug #28076: inconsistent binary/varbinary comparison. 

After adding an index the <VARBINARY> IN (SELECT <BINARY> ...)
clause returned a wrong result: the VARBINARY value was illegally padded
with zero bytes to the length of the BINARY column for the index search.
(<VARBINARY>, ...) IN (SELECT <BINARY>, ... ) clauses are affected too.
parent 00e897ac

mysql-test/r/subselect.result

+49 −0
Original line number Diff line number Diff line
@@ -4150,4 +4150,53 @@ SELECT ((a1,a2) IN (SELECT * FROM t2 WHERE b2 > 0)) IS NULL FROM t1; 
0

0

DROP TABLE t1, t2;

CREATE TABLE t1 (s1 BINARY(5), s2 VARBINARY(5));

INSERT INTO t1 VALUES (0x41,0x41), (0x42,0x42), (0x43,0x43);

SELECT s1, s2 FROM t1 WHERE s2 IN (SELECT s1 FROM t1);

s1	s2

SELECT s1, s2 FROM t1 WHERE (s2, 10) IN (SELECT s1, 10 FROM t1);

s1	s2

CREATE INDEX I1 ON t1 (s1);

CREATE INDEX I2 ON t1 (s2);

SELECT s1, s2 FROM t1 WHERE s2 IN (SELECT s1 FROM t1);

s1	s2

SELECT s1, s2 FROM t1 WHERE (s2, 10) IN (SELECT s1, 10 FROM t1);

s1	s2

TRUNCATE t1;

INSERT INTO t1 VALUES (0x41,0x41);

SELECT * FROM t1 WHERE s1 = (SELECT s2 FROM t1);

s1	s2

DROP TABLE t1;

CREATE TABLE t1 (a1 VARBINARY(2) NOT NULL DEFAULT '0', PRIMARY KEY (a1));

CREATE TABLE t2 (a2 BINARY(2) default '0', INDEX (a2));

CREATE TABLE t3 (a3 BINARY(2) default '0');

INSERT INTO t1 VALUES (1),(2),(3),(4);

INSERT INTO t2 VALUES (1),(2),(3);

INSERT INTO t3 VALUES (1),(2),(3);

SELECT LEFT(t2.a2, 1) FROM t2,t3 WHERE t3.a3=t2.a2;

LEFT(t2.a2, 1)

1

2

3

SELECT t1.a1, t1.a1 in (SELECT t2.a2 FROM t2,t3 WHERE t3.a3=t2.a2) FROM t1;

a1	t1.a1 in (SELECT t2.a2 FROM t2,t3 WHERE t3.a3=t2.a2)

1	0

2	0

3	0

4	0

DROP TABLE t1,t2,t3;

CREATE TABLE t1 (a1 BINARY(3) PRIMARY KEY, b1 VARBINARY(3));

CREATE TABLE t2 (a2 VARBINARY(3) PRIMARY KEY);

CREATE TABLE t3 (a3 VARBINARY(3) PRIMARY KEY);

INSERT INTO t1 VALUES (1,10), (2,20), (3,30), (4,40);

INSERT INTO t2 VALUES (2), (3), (4), (5);

INSERT INTO t3 VALUES (10), (20), (30);

SELECT LEFT(t1.a1,1) FROM t1,t3 WHERE t1.b1=t3.a3;

LEFT(t1.a1,1)

1

2

3

SELECT a2 FROM t2 WHERE t2.a2 IN (SELECT t1.a1 FROM t1,t3 WHERE t1.b1=t3.a3);

a2

DROP TABLE t1, t2, t3;

End of 5.0 tests.

mysql-test/t/subselect.test

+42 −0
Original line number Diff line number Diff line
@@ -3002,4 +3002,46 @@ INSERT INTO t2 VALUES (103, 203); 
SELECT ((a1,a2) IN (SELECT * FROM t2 WHERE b2 > 0)) IS NULL FROM t1;

DROP TABLE t1, t2;



#

# Bug #28076: inconsistent binary/varbinary comparison

#



CREATE TABLE t1 (s1 BINARY(5), s2 VARBINARY(5));

INSERT INTO t1 VALUES (0x41,0x41), (0x42,0x42), (0x43,0x43);



SELECT s1, s2 FROM t1 WHERE s2 IN (SELECT s1 FROM t1);

SELECT s1, s2 FROM t1 WHERE (s2, 10) IN (SELECT s1, 10 FROM t1);



CREATE INDEX I1 ON t1 (s1);

CREATE INDEX I2 ON t1 (s2);



SELECT s1, s2 FROM t1 WHERE s2 IN (SELECT s1 FROM t1);

SELECT s1, s2 FROM t1 WHERE (s2, 10) IN (SELECT s1, 10 FROM t1);



TRUNCATE t1;

INSERT INTO t1 VALUES (0x41,0x41);

SELECT * FROM t1 WHERE s1 = (SELECT s2 FROM t1);



DROP TABLE t1;



CREATE TABLE t1 (a1 VARBINARY(2) NOT NULL DEFAULT '0', PRIMARY KEY (a1));

CREATE TABLE t2 (a2 BINARY(2) default '0', INDEX (a2));

CREATE TABLE t3 (a3 BINARY(2) default '0');

INSERT INTO t1 VALUES (1),(2),(3),(4);

INSERT INTO t2 VALUES (1),(2),(3);

INSERT INTO t3 VALUES (1),(2),(3);

SELECT LEFT(t2.a2, 1) FROM t2,t3 WHERE t3.a3=t2.a2;

SELECT t1.a1, t1.a1 in (SELECT t2.a2 FROM t2,t3 WHERE t3.a3=t2.a2) FROM t1;

DROP TABLE t1,t2,t3;



CREATE TABLE t1 (a1 BINARY(3) PRIMARY KEY, b1 VARBINARY(3));

CREATE TABLE t2 (a2 VARBINARY(3) PRIMARY KEY);

CREATE TABLE t3 (a3 VARBINARY(3) PRIMARY KEY);

INSERT INTO t1 VALUES (1,10), (2,20), (3,30), (4,40);

INSERT INTO t2 VALUES (2), (3), (4), (5);

INSERT INTO t3 VALUES (10), (20), (30);

SELECT LEFT(t1.a1,1) FROM t1,t3 WHERE t1.b1=t3.a3;

SELECT a2 FROM t2 WHERE t2.a2 IN (SELECT t1.a1 FROM t1,t3 WHERE t1.b1=t3.a3);

DROP TABLE t1, t2, t3;



--echo End of 5.0 tests.

sql/item.cc

+12 −4
Original line number Diff line number Diff line
@@ -6259,9 +6259,9 @@ bool field_is_equal_to_item(Field *field,Item *item) 
  return result == field->val_real();

}



Item_cache* Item_cache::get_cache(Item_result type)

Item_cache* Item_cache::get_cache(const Item *item)

{

  switch (type) {

  switch (item->result_type()) {

  case INT_RESULT:

    return new Item_cache_int();

  case REAL_RESULT:
@@ -6269,7 +6269,7 @@ Item_cache* Item_cache::get_cache(Item_result type) 
  case DECIMAL_RESULT:

    return new Item_cache_decimal();

  case STRING_RESULT:

    return new Item_cache_str();

    return new Item_cache_str(item);

  case ROW_RESULT:

    return new Item_cache_row();

  default:
@@ -6447,6 +6447,14 @@ my_decimal *Item_cache_str::val_decimal(my_decimal *decimal_val) 
}





int Item_cache_str::save_in_field(Field *field, bool no_conversions)

{

  int res= Item_cache::save_in_field(field, no_conversions);

  return (is_varbinary && field->type() == MYSQL_TYPE_STRING &&

          value->length() < field->field_length) ? 1 : res;

}





bool Item_cache_row::allocate(uint num)

{

  item_count= num;
@@ -6465,7 +6473,7 @@ bool Item_cache_row::setup(Item * item) 
  {

    Item *el= item->element_index(i);

    Item_cache *tmp;

    if (!(tmp= values[i]= Item_cache::get_cache(el->result_type())))

    if (!(tmp= values[i]= Item_cache::get_cache(el)))

      return 1;

    tmp->setup(el);

  }

sql/item.h

+11 −3
Original line number Diff line number Diff line
@@ -2469,7 +2469,7 @@ class Item_cache: public Item 
  };

  virtual void store(Item *)= 0;

  enum Type type() const { return CACHE_ITEM; }

  static Item_cache* get_cache(Item_result type);

  static Item_cache* get_cache(const Item *item);

  table_map used_tables() const { return used_table_map; }

  virtual void keep_array() {}

  // to prevent drop fixed flag (no need parent cleanup call)
@@ -2531,9 +2531,16 @@ class Item_cache_str: public Item_cache 
{

  char buffer[STRING_BUFFER_USUAL_SIZE];

  String *value, value_buff;

public:

  Item_cache_str(): Item_cache(), value(0) { }

  bool is_varbinary;

  

public:

  Item_cache_str(const Item *item) :

    Item_cache(), value(0),

    is_varbinary(item->type() == FIELD_ITEM &&

                 ((const Item_field *) item)->field->type() ==

                   MYSQL_TYPE_VARCHAR &&

                 !((const Item_field *) item)->field->has_charset())

  {}

  void store(Item *item);

  double val_real();

  longlong val_int();
@@ -2541,6 +2548,7 @@ class Item_cache_str: public Item_cache 
  my_decimal *val_decimal(my_decimal *);

  enum Item_result result_type() const { return STRING_RESULT; }

  CHARSET_INFO *charset() const { return value->charset(); };

  int save_in_field(Field *field, bool no_conversions);

};



class Item_cache_row: public Item_cache

sql/item_cmpfunc.cc

+1 −1
Original line number Diff line number Diff line
@@ -1373,7 +1373,7 @@ longlong Item_func_truth::val_int() 
bool Item_in_optimizer::fix_left(THD *thd, Item **ref)

{

  if (!args[0]->fixed && args[0]->fix_fields(thd, args) ||

      !cache && !(cache= Item_cache::get_cache(args[0]->result_type())))

      !cache && !(cache= Item_cache::get_cache(args[0])))

    return 1;



  cache->setup(args[0]);