Commit e4c09a77 authored by unknown's avatar unknown
Browse files

WL#2466 - Fulltext: "always-index" words 

Trie and Aho-Corasick code added to distribution.


include/Makefile.am:
  my_trie.h added to noinst_HEADERS.
mysys/Makefile.am:
  trie.c added to libmysys_a_SOURCES.
parent 0de81f00

include/Makefile.am

+1 −1
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ noinst_HEADERS = config-win.h config-os2.h config-netware.h \ 
			myisam.h myisampack.h myisammrg.h ft_global.h\

			mysys_err.h my_base.h help_start.h help_end.h \

			my_nosys.h my_alarm.h queues.h rijndael.h sha1.h \

			my_aes.h my_tree.h hash.h thr_alarm.h \

			my_aes.h my_tree.h my_trie.h hash.h thr_alarm.h \

			thr_lock.h t_ctype.h violite.h md5.h \

			mysql_version.h.in my_handler.h my_time.h decimal.h

include/my_trie.h

0 → 100644
+142 −0
Original line number Diff line number Diff line 
/* Copyright (C) 2005 MySQL AB



   This program is free software; you can redistribute it and/or modify

   it under the terms of the GNU General Public License as published by

   the Free Software Foundation; either version 2 of the License, or

   (at your option) any later version.



   This program is distributed in the hope that it will be useful,

   but WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

   GNU General Public License for more details.



   You should have received a copy of the GNU General Public License

   along with this program; if not, write to the Free Software

   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */



#ifndef _trie_h

#define _trie_h

#ifdef	__cplusplus

extern "C" {

#endif



typedef struct st_trie_node

{

  uint16 leaf;                /* Depth from root node if match, 0 else */

  byte c;                     /* Label on this edge */

  struct st_trie_node *next;  /* Next label */

  struct st_trie_node *links; /* Array of edges leaving this node */

  struct st_trie_node *fail;  /* AC failure function */

} TRIE_NODE;



typedef struct st_trie

{

  TRIE_NODE root;

  MEM_ROOT mem_root;

  CHARSET_INFO *charset;

  uint32 nnodes;

  uint32 nwords;

} TRIE;



typedef struct st_ac_trie_state

{

  TRIE *trie;

  TRIE_NODE *node;

} AC_TRIE_STATE;



extern TRIE *trie_init (TRIE *trie, CHARSET_INFO *charset);

extern void trie_free (TRIE *trie);

extern my_bool trie_insert (TRIE *trie, const byte *key, uint keylen);

extern my_bool ac_trie_prepare (TRIE *trie);

extern void ac_trie_init (TRIE *trie, AC_TRIE_STATE *state);





/* `trie_goto' is internal function and shouldn't be used. */



static inline TRIE_NODE *trie_goto (TRIE_NODE *root, TRIE_NODE *node, byte c)

{

  TRIE_NODE *next;

  DBUG_ENTER("trie_goto");

  for (next= node->links; next; next= next->next)

    if (next->c == c)

      DBUG_RETURN(next);

  if (root == node)

    DBUG_RETURN(root);

  DBUG_RETURN(NULL);

}





/*

  SYNOPSIS

    int ac_trie_next (AC_TRIE_STATE *state, byte *c);

    state - valid pointer to `AC_TRIE_STATE'

    c - character to lookup



  DESCRIPTION

    Implementation of search using Aho-Corasick automaton.

    Performs char-by-char search.



  RETURN VALUE

    `ac_trie_next' returns length of matched word or 0.

*/



static inline int ac_trie_next (AC_TRIE_STATE *state, byte *c)

{

  TRIE_NODE *root, *node;

  DBUG_ENTER("ac_trie_next");

  DBUG_ASSERT(state && c);

  root= &state->trie->root;

  node= state->node;

  while (! (state->node= trie_goto(root, node, *c)))

    node= node->fail;

  DBUG_RETURN(state->node->leaf);

}





/*

  SYNOPSIS

    my_bool trie_search (TRIE *trie, const byte *key, uint keylen);

    trie - valid pointer to `TRIE'

    key - valid pointer to key to insert

    keylen - non-0 key length



  DESCRIPTION

    Performs key lookup in trie.



  RETURN VALUE

    `trie_search' returns `true' if key is in `trie'. Otherwise,

    `false' is returned.



  NOTES

    Consecutive search here is "best by test". arrays are very short, so

    binary search or hashing would add too much complexity that would

    overweight speed gain. Especially because compiler can optimize simple

    consecutive loop better (tested)

*/



static inline my_bool trie_search (TRIE *trie, const byte *key, uint keylen)

{

  TRIE_NODE *node;

  uint k;

  DBUG_ENTER("trie_search");

  DBUG_ASSERT(trie && key && keylen);

  node= &trie->root;



  for (k= 0; k < keylen; k++)

  {

    byte p;

    if (! (node= node->links))

      DBUG_RETURN(FALSE);

    p= key[k];

    while (p != node->c)

      if (! (node= node->next))

        DBUG_RETURN(FALSE);

  }



  DBUG_RETURN(node->leaf > 0);

}



#ifdef	__cplusplus

}

#endif

#endif

mysys/Makefile.am

+1 −1
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ libmysys_a_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ 
			mf_wcomp.c mf_wfile.c my_gethwaddr.c \

			mf_qsort.c mf_qsort2.c mf_sort.c \

			ptr_cmp.c mf_radix.c queues.c \

			tree.c list.c hash.c array.c string.c typelib.c \

			tree.c trie.c list.c hash.c array.c string.c typelib.c \

			my_copy.c my_append.c my_lib.c \

			my_delete.c my_rename.c my_redel.c my_tempnam.c \

			my_chsize.c my_lread.c my_lwrite.c my_clock.c \

mysys/trie.c

0 → 100644
+237 −0
Original line number Diff line number Diff line 
/* Copyright (C) 2005 MySQL AB



   This program is free software; you can redistribute it and/or modify

   it under the terms of the GNU General Public License as published by

   the Free Software Foundation; either version 2 of the License, or

   (at your option) any later version.



   This program is distributed in the hope that it will be useful,

   but WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

   GNU General Public License for more details.



   You should have received a copy of the GNU General Public License

   along with this program; if not, write to the Free Software

   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */



/*

  Implementation of trie and Aho-Corasick automaton.

  Supports only charsets that can be compared byte-wise.

  

  TODO:

    Add character frequencies. Can increase lookup speed

    up to 30%.

    Implement character-wise comparision.

*/





#include "mysys_priv.h"

#include <m_string.h>

#include <my_trie.h>

#include <my_base.h>





/*

  SYNOPSIS

    TRIE *trie_init (TRIE *trie, CHARSET_INFO *charset);



  DESCRIPTION

    Allocates or initializes a `TRIE' object. If `trie' is a `NULL'

    pointer, the function allocates, initializes, and returns a new

    object. Otherwise, the object is initialized and the address of

    the object is returned.  If `trie_init()' allocates a new object,

    it will be freed when `trie_free()' is called.



  RETURN VALUE

    An initialized `TRIE*' object.  `NULL' if there was insufficient

    memory to allocate a new object.

*/



TRIE *trie_init (TRIE *trie, CHARSET_INFO *charset)

{

  MEM_ROOT mem_root;

  DBUG_ENTER("trie_init");

  DBUG_ASSERT(charset);

  init_alloc_root(&mem_root,

                  (sizeof(TRIE_NODE) * 128) + ALLOC_ROOT_MIN_BLOCK_SIZE,

                  sizeof(TRIE_NODE) * 128);

  if (! trie)

  {

    if (! (trie= (TRIE *)alloc_root(&mem_root, sizeof(TRIE))))

    {

      free_root(&mem_root, MYF(0));

      DBUG_RETURN(NULL);

    }

  }



  memcpy(&trie->mem_root, &mem_root, sizeof(MEM_ROOT));

  trie->root.leaf= 0;

  trie->root.c= 0;

  trie->root.next= NULL;

  trie->root.links= NULL;

  trie->root.fail= NULL;

  trie->charset= charset;

  trie->nnodes= 0;

  trie->nwords= 0;

  DBUG_RETURN(trie);

}





/*

  SYNOPSIS

    void trie_free (TRIE *trie);

    trie - valid pointer to `TRIE'



  DESCRIPTION

    Frees the memory allocated for a `trie'.



  RETURN VALUE

    None.

*/



void trie_free (TRIE *trie)

{

  MEM_ROOT mem_root;

  DBUG_ENTER("trie_free");

  DBUG_ASSERT(trie);

  memcpy(&mem_root, &trie->mem_root, sizeof(MEM_ROOT));

  free_root(&mem_root, MYF(0));

  DBUG_VOID_RETURN;

}





/*

  SYNOPSIS

    my_bool trie_insert (TRIE *trie, const byte *key, uint keylen);

    trie - valid pointer to `TRIE'

    key - valid pointer to key to insert

    keylen - non-0 key length



  DESCRIPTION

    Inserts new key into trie.



  RETURN VALUE

    Upon successful completion, `trie_insert' returns `FALSE'. Otherwise

    `TRUE' is returned.



  NOTES

    If this function fails you must assume `trie' is broken.

    However it can be freed with trie_free().

*/



my_bool trie_insert (TRIE *trie, const byte *key, uint keylen)

{

  TRIE_NODE *node;

  TRIE_NODE *next;

  byte p;

  uint k;

  DBUG_ENTER("trie_insert");

  DBUG_ASSERT(trie && key && keylen);

  node= &trie->root;

  trie->root.fail= NULL;

  for (k= 0; k < keylen; k++)

  {

    p= key[k];

    for (next= node->links; next; next= next->next)

      if (next->c == p)

        break;



    if (! next)

    {

      TRIE_NODE *tmp= (TRIE_NODE *)alloc_root(&trie->mem_root,

                                              sizeof(TRIE_NODE));

      if (! tmp)

        DBUG_RETURN(TRUE);

      tmp->leaf= 0;

      tmp->c= p;

      tmp->links= tmp->fail= tmp->next= NULL;

      trie->nnodes++;

      if (! node->links)

      {

        node->links= tmp;

      }

      else

      {

        for (next= node->links; next->next; next= next->next) /* no-op */;

        next->next= tmp;

      }

      node= tmp;

    }

    else

    {

      node= next;

    }

  }

  node->leaf= keylen;

  trie->nwords++;

  DBUG_RETURN(FALSE);

}





/*

  SYNOPSIS

    my_bool trie_prepare (TRIE *trie);

    trie - valid pointer to `TRIE'



  DESCRIPTION

    Constructs Aho-Corasick automaton.



  RETURN VALUE

    Upon successful completion, `trie_prepare' returns `FALSE'. Otherwise

    `TRUE' is returned.

*/



my_bool ac_trie_prepare (TRIE *trie)

{

  TRIE_NODE **tmp_nodes;

  TRIE_NODE *node;

  uint32 fnode= 0;

  uint32 lnode= 0;

  DBUG_ENTER("trie_prepare");

  DBUG_ASSERT(trie);



  tmp_nodes= (TRIE_NODE **)my_malloc(trie->nnodes * sizeof(TRIE_NODE *), MYF(0));

  if (! tmp_nodes)

    DBUG_RETURN(TRUE);



  trie->root.fail= &trie->root;

  for (node= trie->root.links; node; node= node->next)

  {

    node->fail= &trie->root;

    tmp_nodes[lnode++]= node;

  }



  while (fnode < lnode)

  {

    TRIE_NODE *current= (TRIE_NODE *)tmp_nodes[fnode++];

    for (node= current->links; node; node= node->next)

    {

      TRIE_NODE *fail= current->fail;

      tmp_nodes[lnode++]= node;

      while (! (node->fail= trie_goto(&trie->root, fail, node->c)))

        fail= fail->fail;

    }

  }

  my_free((gptr)tmp_nodes, MYF(0));

  DBUG_RETURN(FALSE);

}





/*

  SYNOPSIS

    void ac_trie_init (TRIE *trie, AC_TRIE_STATE *state);

    trie - valid pointer to `TRIE'

    state - value pointer to `AC_TRIE_STATE'



  DESCRIPTION

    Initializes `AC_TRIE_STATE' object.

*/



void ac_trie_init (TRIE *trie, AC_TRIE_STATE *state)

{

  DBUG_ENTER("ac_trie_init");

  DBUG_ASSERT(trie && state);

  state->trie= trie;

  state->node= &trie->root;

  DBUG_VOID_RETURN;

}