Commit 15a13203 authored by unknown's avatar unknown
Browse files

Now ftparser does not need to bother about memory management - 

it can tell MySQL to make a copy of everything (bug#17123)


include/mysql/plugin.h:
  Now ftparser does not need to bother about memory management -
  it can tell MySQL to make a copy of everything (bug#17123)
  MYSQL_FTFLAGS_NEED_COPY flag
storage/myisam/ft_boolean_search.c:
  param->flags
storage/myisam/ft_nlq_search.c:
  param->flags. ft_parse takes a mem_root as an argument
storage/myisam/ft_parser.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/ft_update.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/ftdefs.h:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/mi_check.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary
storage/myisam/myisamdef.h:
  memroot for ftparser in MI_INFO and MI_SORT_PARAM
storage/myisam/sort.c:
  free ftparser memroot
parent f1bb09e7

include/mysql/plugin.h

+15 −1
Original line number Diff line number Diff line
@@ -201,6 +201,17 @@ typedef struct st_mysql_ftparser_boolean_info 
  char *quot;

} MYSQL_FTPARSER_BOOLEAN_INFO;



/*

  The following flag means that buffer with a string (document, word)

  may be overwritten by the caller before the end of the parsing (that is

  before st_mysql_ftparser::deinit() call). If one needs the string

  to survive between two successive calls of the parsing function, she

  needs to save a copy of it. The flag may be set by MySQL before calling

  st_mysql_ftparser::parse(), or it may be set by a plugin before calling

  st_mysql_ftparser_param::mysql_parse() or

  st_mysql_ftparser_param::mysql_add_word().

*/

#define MYSQL_FTFLAGS_NEED_COPY 1



/*

  An argument of the full-text parser plugin. This structure is
@@ -234,8 +245,10 @@ typedef struct st_mysql_ftparser_boolean_info 


  length: Length of the document or query string, in bytes.



  flags: See MYSQL_FTFLAGS_* constants above.



  mode: The parsing mode.  With boolean operators, with stopwords, or

  nothing.  See MYSQL_FTPARSER_* constants above.

  nothing.  See  enum_ftparser_mode above.

*/



typedef struct st_mysql_ftparser_param
@@ -250,6 +263,7 @@ typedef struct st_mysql_ftparser_param 
  struct charset_info_st *cs;

  char *doc;

  int length;

  int flags;

  enum enum_ftparser_mode mode;

} MYSQL_FTPARSER_PARAM;

storage/myisam/ft_boolean_search.c

+3 −0
Original line number Diff line number Diff line
@@ -311,6 +311,7 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len, 
  param->cs= ftb->charset;

  param->doc= query;

  param->length= len;

  param->flags= 0;

  param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;

  parser->parse(param);

  DBUG_VOID_RETURN;
@@ -665,6 +666,7 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len, 
  param->cs= ftb->charset;

  param->doc= (byte *)document;

  param->length= len;

  param->flags= 0;

  param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;

  parser->parse(param);

  DBUG_RETURN(ftb_param.match ? 1 : 0);
@@ -916,6 +918,7 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length) 
  param->mysql_parse= ftb_find_relevance_parse;

  param->mysql_add_word= ftb_find_relevance_add_word;

  param->mysql_ftparam= (void *)&ftb_param;

  param->flags= 0;

  param->cs= ftb->charset;

  param->mode= MYSQL_FTPARSER_SIMPLE_MODE;

  while (_mi_ft_segiterator(&ftsi))

storage/myisam/ft_nlq_search.c

+6 −2
Original line number Diff line number Diff line
@@ -235,7 +235,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query, 
            NULL, NULL);



  ft_parse_init(&wtree, aio.charset);

  if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))

  ftparser_param->flags= 0;

  if (ft_parse(&wtree, query, query_len, parser, ftparser_param,

               &wtree.mem_root))

    goto err;



  if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
@@ -255,7 +257,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query, 
      if (!(*info->read_record)(info,docid,record))

      {

        info->update|= HA_STATE_AKTIV;

        _mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);

        ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;

        _mi_ft_parse(&wtree, info, keynr, record, ftparser_param,

                     &wtree.mem_root);

      }

    }

    delete_queue(&best);

storage/myisam/ft_parser.c

+12 −13
Original line number Diff line number Diff line
@@ -24,14 +24,12 @@ typedef struct st_ft_docstat { 
  double sum;

} FT_DOCSTAT;





typedef struct st_my_ft_parser_param

{

  TREE     *wtree;

  my_bool with_alloc;

  MEM_ROOT *mem_root;

} MY_FT_PARSER_PARAM;





static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)

{

  return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
@@ -48,14 +46,14 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) 


/* transforms tree of words into the array, applying normalization */



FT_WORD * ft_linearize(TREE *wtree)

FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root)

{

  FT_WORD *wlist,*p;

  FT_DOCSTAT docstat;

  DBUG_ENTER("ft_linearize");



  if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)*

				   (1+wtree->elements_in_tree),MYF(0))))

  if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*

                                    (1+wtree->elements_in_tree))))

  {

    docstat.list=wlist;

    docstat.uniq=wtree->elements_in_tree;
@@ -249,12 +247,11 @@ static int ft_add_word(MYSQL_FTPARSER_PARAM *param, 
  MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;

  DBUG_ENTER("ft_add_word");

  wtree= ft_param->wtree;

  if (ft_param->with_alloc)

  if (param->flags & MYSQL_FTFLAGS_NEED_COPY)

  {

    byte *ptr;

    /* allocating the data in the tree - to avoid mallocs and frees */

    DBUG_ASSERT(wtree->with_delete == 0);

    ptr= (byte *)alloc_root(&wtree->mem_root, word_len);

    ptr= (byte *)alloc_root(ft_param->mem_root, word_len);

    memcpy(ptr, word, word_len);

    w.pos= ptr;

  }
@@ -286,16 +283,16 @@ static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param, 
}





int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,

int ft_parse(TREE *wtree, byte *doc, int doclen,

                    struct st_mysql_ftparser *parser,

                    MYSQL_FTPARSER_PARAM *param)

                    MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)

{

  MY_FT_PARSER_PARAM my_param;

  DBUG_ENTER("ft_parse");

  DBUG_ASSERT(parser);



  my_param.wtree= wtree;

  my_param.with_alloc= with_alloc;

  my_param.mem_root= mem_root;



  param->mysql_parse= ft_parse_internal;

  param->mysql_add_word= ft_add_word;
@@ -357,6 +354,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, 
    info->ftparser_param= (MYSQL_FTPARSER_PARAM *)

      my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *

                info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));

    init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);

    if (! info->ftparser_param)

      return 0;

  }
@@ -388,6 +386,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, 
void ftparser_call_deinitializer(MI_INFO *info)

{

  uint i, j, keys= info->s->state.header.keys;

  free_root(&info->ft_memroot, MYF(0));

  if (! info->ftparser_param)

    return;

  for (i= 0; i < keys; i++)

storage/myisam/ft_update.c

+26 −28
Original line number Diff line number Diff line
@@ -95,9 +95,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi) 


/* parses a document i.e. calls ft_parse for every keyseg */



uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,

                  const byte *record, my_bool with_alloc,

                  MYSQL_FTPARSER_PARAM *param)

uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record,

                  MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)

{

  FT_SEG_ITERATOR ftsi;

  struct st_mysql_ftparser *parser;
@@ -110,14 +109,14 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, 
  while (_mi_ft_segiterator(&ftsi))

  {

    if (ftsi.pos)

      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,

                   param))

      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, parser, param, mem_root))

        DBUG_RETURN(1);

  }

  DBUG_RETURN(0);

}



FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)

FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record,

                             MEM_ROOT *mem_root)

{

  TREE ptree;

  MYSQL_FTPARSER_PARAM *param;
@@ -125,10 +124,11 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record) 
  if (! (param= ftparser_call_initializer(info, keynr, 0)))

    DBUG_RETURN(NULL);

  bzero((char*) &ptree, sizeof(ptree));

  if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))

  param->flags= 0;

  if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root))

    DBUG_RETURN(NULL);



  DBUG_RETURN(ft_linearize(&ptree));

  DBUG_RETURN(ft_linearize(&ptree, mem_root));

}



static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf,
@@ -206,10 +206,11 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, 
  int cmp, cmp2;

  DBUG_ENTER("_mi_ft_update");



  if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec)))

    goto err0;

  if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec)))

    goto err1;

  if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec,

                                            &info->ft_memroot)) ||

      !(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec,

                                            &info->ft_memroot)))

    goto err;



  error=0;

  while(old_word->pos && new_word->pos)
@@ -222,13 +223,13 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, 
    {

      key_length=_ft_make_key(info,keynr,keybuf,old_word,pos);

      if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)))

        goto err2;

        goto err;

    }

    if (cmp > 0 || cmp2)

    {

      key_length=_ft_make_key(info,keynr,keybuf,new_word,pos);

      if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)))

        goto err2;

        goto err;

    }

    if (cmp<=0) old_word++;

    if (cmp>=0) new_word++;
@@ -238,11 +239,8 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, 
 else if (new_word->pos)

   error=_mi_ft_store(info,keynr,keybuf,new_word,pos);



err2:

    my_free((char*) newlist,MYF(0));

err1:

    my_free((char*) oldlist,MYF(0));

err0:

err:

  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));

  DBUG_RETURN(error);

}
@@ -255,12 +253,13 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, 
  int error= -1;

  FT_WORD *wlist;

  DBUG_ENTER("_mi_ft_add");

  DBUG_PRINT("enter",("keynr: %d",keynr));



  if ((wlist=_mi_ft_parserecord(info, keynr, record)))

  {

  if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))

    error=_mi_ft_store(info,keynr,keybuf,wlist,pos);

    my_free((char*) wlist,MYF(0));

  }



  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));

  DBUG_PRINT("exit",("Return: %d",error));

  DBUG_RETURN(error);

}
@@ -275,11 +274,10 @@ int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, 
  DBUG_ENTER("_mi_ft_del");

  DBUG_PRINT("enter",("keynr: %d",keynr));



  if ((wlist=_mi_ft_parserecord(info, keynr, record)))

  {

  if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))

    error=_mi_ft_erase(info,keynr,keybuf,wlist,pos);

    my_free((char*) wlist,MYF(0));

  }



  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));

  DBUG_PRINT("exit",("Return: %d",error));

  DBUG_RETURN(error);

}