Commit 7063bd4d authored by unknown's avatar unknown
Browse files

Bug#15377 Valid multibyte sequences are truncated on INSERT 

ctype-euc_kr.c:
ctype-gb2312.c:
  Adding specific well_formed_length functions
  for gb2312 and euckr, to allow storing characters
  which are correct according to the character set
  specifications but just don't have Unicode mapping.
  Previously only those which have Unicode mapping
  could be stored, while unassigned characters lead
  to data truncation.
Many files:
  new file


strings/ctype-gb2312.c:
  Bug#15377 Valid multibyte sequences are truncated on INSERT
  Adding specific well_formed_length functions
  for gb2312 and euckr, to allow storing characters
  which are correct according to the character set.
  Previously only those which have Unicode mapping
  could be stored.
strings/ctype-euc_kr.c:
  Adding specific well_formed_length functions
  for gb2312 and euckr, to allow storing characters
  which are correct according to the character set.
  Previously only those which have Unicode mapping
  could be stored.
parent 5aeb6929

mysql-test/include/have_euckr.inc

0 → 100644
+4 −0
Original line number Diff line number Diff line 
-- require r/have_euckr.require

disable_query_log;

show collation like "euckr_korean_ci";

enable_query_log;

mysql-test/include/have_gb2312.inc

0 → 100644
+4 −0
Original line number Diff line number Diff line 
-- require r/have_gb2312.require

disable_query_log;

show collation like "gb2312_chinese_ci";

enable_query_log;

mysql-test/r/ctype_euckr.result

0 → 100644
+167 −0
Original line number Diff line number Diff line 
drop table if exists t1;

SET @test_character_set= 'euckr';

SET @test_collation= 'euckr_korean_ci';

SET @safe_character_set_server= @@character_set_server;

SET @safe_collation_server= @@collation_server;

SET character_set_server= @test_character_set;

SET collation_server= @test_collation;

CREATE DATABASE d1;

USE d1;

CREATE TABLE t1 (c CHAR(10), KEY(c));

SHOW FULL COLUMNS FROM t1;

Field	Type	Collation	Null	Key	Default	Extra	Privileges	Comment

c	char(10)	euckr_korean_ci	YES	MUL	NULL			

INSERT INTO t1 VALUES ('aaa'),('aaaa'),('aaaaa');

SELECT c as want3results FROM t1 WHERE c LIKE 'aaa%';

want3results

aaa

aaaa

aaaaa

DROP TABLE t1;

CREATE TABLE t1 (c1 varchar(15), KEY c1 (c1(2)));

SHOW FULL COLUMNS FROM t1;

Field	Type	Collation	Null	Key	Default	Extra	Privileges	Comment

c1	varchar(15)	euckr_korean_ci	YES	MUL	NULL			

INSERT INTO t1 VALUES ('location'),('loberge'),('lotre'),('boabab');

SELECT c1 as want3results from t1 where c1 like 'l%';

want3results

location

loberge

lotre

SELECT c1 as want3results from t1 where c1 like 'lo%';

want3results

location

loberge

lotre

SELECT c1 as want1result  from t1 where c1 like 'loc%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'loca%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locat%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locati%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locatio%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'location%';

want1result

location

DROP TABLE t1;

DROP DATABASE d1;

USE test;

SET character_set_server= @safe_character_set_server;

SET collation_server= @safe_collation_server;

SET NAMES euckr;

SET collation_connection='euckr_korean_ci';

create table t1 select repeat('a',4000) a;

delete from t1;

insert into t1 values ('a'), ('a '), ('a\t');

select collation(a),hex(a) from t1 order by a;

collation(a)	hex(a)

euckr_korean_ci	6109

euckr_korean_ci	61

euckr_korean_ci	6120

drop table t1;

create table t1 engine=innodb select repeat('a',50) as c1;

alter table t1 add index(c1(5));

insert into t1 values ('abcdefg'),('abcde100'),('abcde110'),('abcde111');

select collation(c1) from t1 limit 1;

collation(c1)

euckr_korean_ci

select c1 from t1 where c1 like 'abcdef%' order by c1;

c1

abcdefg

select c1 from t1 where c1 like 'abcde1%' order by c1;

c1

abcde100

abcde110

abcde111

select c1 from t1 where c1 like 'abcde11%' order by c1;

c1

abcde110

abcde111

select c1 from t1 where c1 like 'abcde111%' order by c1;

c1

abcde111

drop table t1;

select @@collation_connection;

@@collation_connection

euckr_korean_ci

create table t1 ROW_FORMAT=DYNAMIC select repeat('a',50) as c1 ;

insert into t1 values('abcdef');

insert into t1 values('_bcdef');

insert into t1 values('a_cdef');

insert into t1 values('ab_def');

insert into t1 values('abc_ef');

insert into t1 values('abcd_f');

insert into t1 values('abcde_');

select c1 as c1u from t1 where c1 like 'ab\_def';

c1u

ab_def

select c1 as c2h from t1 where c1 like 'ab#_def' escape '#';

c2h

ab_def

drop table t1;

SET collation_connection='euckr_bin';

create table t1 select repeat('a',4000) a;

delete from t1;

insert into t1 values ('a'), ('a '), ('a\t');

select collation(a),hex(a) from t1 order by a;

collation(a)	hex(a)

euckr_bin	6109

euckr_bin	61

euckr_bin	6120

drop table t1;

create table t1 engine=innodb select repeat('a',50) as c1;

alter table t1 add index(c1(5));

insert into t1 values ('abcdefg'),('abcde100'),('abcde110'),('abcde111');

select collation(c1) from t1 limit 1;

collation(c1)

euckr_bin

select c1 from t1 where c1 like 'abcdef%' order by c1;

c1

abcdefg

select c1 from t1 where c1 like 'abcde1%' order by c1;

c1

abcde100

abcde110

abcde111

select c1 from t1 where c1 like 'abcde11%' order by c1;

c1

abcde110

abcde111

select c1 from t1 where c1 like 'abcde111%' order by c1;

c1

abcde111

drop table t1;

select @@collation_connection;

@@collation_connection

euckr_bin

create table t1 ROW_FORMAT=DYNAMIC select repeat('a',50) as c1 ;

insert into t1 values('abcdef');

insert into t1 values('_bcdef');

insert into t1 values('a_cdef');

insert into t1 values('ab_def');

insert into t1 values('abc_ef');

insert into t1 values('abcd_f');

insert into t1 values('abcde_');

select c1 as c1u from t1 where c1 like 'ab\_def';

c1u

ab_def

select c1 as c2h from t1 where c1 like 'ab#_def' escape '#';

c2h

ab_def

drop table t1;

SET NAMES euckr;

CREATE TABLE t1 (a text) character set euckr;

INSERT INTO t1 VALUES (0xA2E6),(0xFEF7);

SELECT hex(a) FROM t1 ORDER BY a;

hex(a)

A2E6

FEF7

DROP TABLE t1;

mysql-test/r/ctype_gb2312.result

0 → 100644
+167 −0
Original line number Diff line number Diff line 
drop table if exists t1;

SET @test_character_set= 'gb2312';

SET @test_collation= 'gb2312_chinese_ci';

SET @safe_character_set_server= @@character_set_server;

SET @safe_collation_server= @@collation_server;

SET character_set_server= @test_character_set;

SET collation_server= @test_collation;

CREATE DATABASE d1;

USE d1;

CREATE TABLE t1 (c CHAR(10), KEY(c));

SHOW FULL COLUMNS FROM t1;

Field	Type	Collation	Null	Key	Default	Extra	Privileges	Comment

c	char(10)	gb2312_chinese_ci	YES	MUL	NULL			

INSERT INTO t1 VALUES ('aaa'),('aaaa'),('aaaaa');

SELECT c as want3results FROM t1 WHERE c LIKE 'aaa%';

want3results

aaa

aaaa

aaaaa

DROP TABLE t1;

CREATE TABLE t1 (c1 varchar(15), KEY c1 (c1(2)));

SHOW FULL COLUMNS FROM t1;

Field	Type	Collation	Null	Key	Default	Extra	Privileges	Comment

c1	varchar(15)	gb2312_chinese_ci	YES	MUL	NULL			

INSERT INTO t1 VALUES ('location'),('loberge'),('lotre'),('boabab');

SELECT c1 as want3results from t1 where c1 like 'l%';

want3results

location

loberge

lotre

SELECT c1 as want3results from t1 where c1 like 'lo%';

want3results

location

loberge

lotre

SELECT c1 as want1result  from t1 where c1 like 'loc%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'loca%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locat%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locati%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'locatio%';

want1result

location

SELECT c1 as want1result  from t1 where c1 like 'location%';

want1result

location

DROP TABLE t1;

DROP DATABASE d1;

USE test;

SET character_set_server= @safe_character_set_server;

SET collation_server= @safe_collation_server;

SET NAMES gb2312;

SET collation_connection='gb2312_chinese_ci';

create table t1 select repeat('a',4000) a;

delete from t1;

insert into t1 values ('a'), ('a '), ('a\t');

select collation(a),hex(a) from t1 order by a;

collation(a)	hex(a)

gb2312_chinese_ci	6109

gb2312_chinese_ci	61

gb2312_chinese_ci	6120

drop table t1;

create table t1 engine=innodb select repeat('a',50) as c1;

alter table t1 add index(c1(5));

insert into t1 values ('abcdefg'),('abcde100'),('abcde110'),('abcde111');

select collation(c1) from t1 limit 1;

collation(c1)

gb2312_chinese_ci

select c1 from t1 where c1 like 'abcdef%' order by c1;

c1

abcdefg

select c1 from t1 where c1 like 'abcde1%' order by c1;

c1

abcde100

abcde110

abcde111

select c1 from t1 where c1 like 'abcde11%' order by c1;

c1

abcde110

abcde111

select c1 from t1 where c1 like 'abcde111%' order by c1;

c1

abcde111

drop table t1;

select @@collation_connection;

@@collation_connection

gb2312_chinese_ci

create table t1 ROW_FORMAT=DYNAMIC select repeat('a',50) as c1 ;

insert into t1 values('abcdef');

insert into t1 values('_bcdef');

insert into t1 values('a_cdef');

insert into t1 values('ab_def');

insert into t1 values('abc_ef');

insert into t1 values('abcd_f');

insert into t1 values('abcde_');

select c1 as c1u from t1 where c1 like 'ab\_def';

c1u

ab_def

select c1 as c2h from t1 where c1 like 'ab#_def' escape '#';

c2h

ab_def

drop table t1;

SET collation_connection='gb2312_bin';

create table t1 select repeat('a',4000) a;

delete from t1;

insert into t1 values ('a'), ('a '), ('a\t');

select collation(a),hex(a) from t1 order by a;

collation(a)	hex(a)

gb2312_bin	6109

gb2312_bin	61

gb2312_bin	6120

drop table t1;

create table t1 engine=innodb select repeat('a',50) as c1;

alter table t1 add index(c1(5));

insert into t1 values ('abcdefg'),('abcde100'),('abcde110'),('abcde111');

select collation(c1) from t1 limit 1;

collation(c1)

gb2312_bin

select c1 from t1 where c1 like 'abcdef%' order by c1;

c1

abcdefg

select c1 from t1 where c1 like 'abcde1%' order by c1;

c1

abcde100

abcde110

abcde111

select c1 from t1 where c1 like 'abcde11%' order by c1;

c1

abcde110

abcde111

select c1 from t1 where c1 like 'abcde111%' order by c1;

c1

abcde111

drop table t1;

select @@collation_connection;

@@collation_connection

gb2312_bin

create table t1 ROW_FORMAT=DYNAMIC select repeat('a',50) as c1 ;

insert into t1 values('abcdef');

insert into t1 values('_bcdef');

insert into t1 values('a_cdef');

insert into t1 values('ab_def');

insert into t1 values('abc_ef');

insert into t1 values('abcd_f');

insert into t1 values('abcde_');

select c1 as c1u from t1 where c1 like 'ab\_def';

c1u

ab_def

select c1 as c2h from t1 where c1 like 'ab#_def' escape '#';

c2h

ab_def

drop table t1;

SET NAMES gb2312;

CREATE TABLE t1 (a text) character set gb2312;

INSERT INTO t1 VALUES (0xA2A1),(0xD7FE);

SELECT hex(a) FROM t1 ORDER BY a;

hex(a)

A2A1

D7FE

DROP TABLE t1;

mysql-test/r/have_euckr.require

0 → 100644
+2 −0
Original line number Diff line number Diff line 
Collation	Charset	Id	Default	Compiled	Sortlen

euckr_korean_ci	euckr	19	Yes	Yes	1