Commit 3d3ef652 authored by serg@serg.mysql.com's avatar serg@serg.mysql.com
Browse files

Boolean search passes _some_ tests

parent 5a9a3e91

.bzrignore

+2 −0
Original line number Diff line number Diff line
@@ -412,3 +412,5 @@ libmysqld/examples/sql_string.cc 
libmysqld/examples/sql_string.h

libmysqld/examples/mysql

libmysqld/examples/mysqltest

myisam/FT1.MYD

myisam/FT1.MYI

include/ft_global.h

+18 −17
Original line number Diff line number Diff line
@@ -29,17 +29,21 @@ extern "C" { 
#define FT_QUERY_MAXLEN 1024

#define HA_FT_MAXLEN 254



typedef struct ft_doc_rec {

  my_off_t  dpos;

  double    weight;

} FT_DOC;



typedef struct st_ft_doclist {

  int       ndocs;

  int       curdoc;

  void      *info;  /* actually (MI_INFO *) but don't want to include myisam.h */

  FT_DOC    doc[1];

} FT_DOCLIST;

typedef struct st_ft_info FT_INFO;

struct _ft_vft {

  int       (*read_next)(FT_INFO *, char *);

  float     (*find_relevance)(FT_INFO *, my_off_t);

  void      (*close_search)(FT_INFO *);

  float     (*get_relevance)(FT_INFO *);

  my_off_t  (*get_docid)(FT_INFO *);

  void      (*reinit_search)(FT_INFO *);

};



#ifndef FT_CORE

struct st_ft_info {

  struct _ft_vft *please; /* INTERCAL style :-) */

};

#endif



extern const char *ft_precompiled_stopwords[];
@@ -50,12 +54,9 @@ extern uint ft_max_word_len_for_sort; 
int ft_init_stopwords(const char **);

void ft_free_stopwords(void);



FT_DOCLIST * ft_nlq_init_search(void *, uint, byte *, uint, my_bool);

int          ft_nlq_read_next(FT_DOCLIST *, char *);

#define      ft_nlq_close_search(handler)    my_free(((gptr)(handler)),MYF(0))

#define      ft_nlq_get_relevance(handler)   (((FT_DOCLIST *)(handler))->doc[((FT_DOCLIST *)(handler))->curdoc].weight)

#define      ft_nlq_get_docid(handler)       (((FT_DOCLIST *)(handler))->doc[((FT_DOCLIST *)(handler))->curdoc].dpos)

#define      ft_nlq_reinit_search(handler)   (((FT_DOCLIST *)(handler))->curdoc=-1)

#define FT_NL  0

#define FT_BOOL 1

FT_INFO *ft_init_search(uint,void *, uint, byte *, uint, my_bool);



#ifdef  __cplusplus

}

include/my_base.h

+1 −0
Original line number Diff line number Diff line
@@ -159,6 +159,7 @@ enum ha_base_keytype { 
#define HA_BLOB_PART		 32

#define HA_SWAP_KEY		 64

#define HA_REVERSE_SORT		 128	/* Sort key in reverse order */

#define HA_NO_SORT               256 /* do not bother sorting on this keyseg */



	/* optionbits for database */

#define HA_OPTION_PACK_RECORD		1

include/my_global.h

+2 −1
Original line number Diff line number Diff line
@@ -262,7 +262,7 @@ int __void__; 
#define LINT_INIT(var)

#endif



/* Define som useful general macros */

/* Define some useful general macros */

#if defined(__cplusplus) && defined(__GNUC__)

#define max(a, b)	((a) >? (b))

#define min(a, b)	((a) <? (b))
@@ -276,6 +276,7 @@ typedef unsigned int uint; 
typedef unsigned short ushort;

#endif



#define comp(a,b)	(((a) < (b)) ? -1 : ((a) > (b)) ? 1 : 0)

#define sgn(a)		(((a) < 0) ? -1 : ((a) > 0) ? 1 : 0)

#define swap(t,a,b)	{ register t dummy; dummy = a; a = b; b = dummy; }

#define test(a)		((a) ? 1 : 0)

myisam/ft_boolean_search.c

+137 −103
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


/* Written by Sergei A. Golubchik, who has a shared copyright to this code */



#define FT_CORE

#include "ftdefs.h"

#include <queues.h>
@@ -73,7 +74,8 @@ typedef struct { 
  byte      word[1];

} FTB_WORD;



typedef struct st_ftb_handler {

typedef struct st_ft_info {

  struct _ft_vft *please;

  MI_INFO  *info;

  uint       keynr;

  int        ok;
@@ -85,10 +87,10 @@ typedef struct st_ftb_handler { 
int FTB_WORD_cmp(void *v, byte *a, byte *b)

{

  /* ORDER BY docid, ndepth DESC */

  int i=((FTB_WORD *)a)->docid-((FTB_WORD *)b)->docid;

  int i=comp(((FTB_WORD *)a)->docid, ((FTB_WORD *)b)->docid);

  if (!i)

    i=((FTB_WORD *)b)->ndepth-((FTB_WORD *)a)->ndepth;

  return sgn(i);

    i=comp(((FTB_WORD *)b)->ndepth,((FTB_WORD *)a)->ndepth);

  return i;

}



void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
@@ -108,7 +110,8 @@ void _ftb_parse_query(FTB *ftb, byte **start, byte *end, 
  if (! ftb->ok)

    return;



  while (res=ftb_get_word(&start,end,&w,&param))

  param.prev=' ';

  while (res=ft_get_word(start,end,&w,&param))

  {

    byte  r=param.plusminus;

    float weight=(param.pmsign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)];
@@ -170,8 +173,8 @@ void _ftb_parse_query(FTB *ftb, byte **start, byte *end, 
  return;

}



FTB * ft_boolean_search_init(MI_INFO *info, uint keynr, byte *query,

                               uint query_len)

FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,

                    uint query_len, my_bool presort __attribute__((unused)))

{

  FTB       *ftb;

  FTB_EXPR  *ftbe;
@@ -179,11 +182,12 @@ FTB * ft_boolean_search_init(MI_INFO *info, uint keynr, byte *query, 


  if (!(ftb=(FTB *)my_malloc(sizeof(FTB), MYF(MY_WME))))

    return 0;

  ftb->please=& _ft_vft_boolean;

  ftb->ok=1;

  ftb->info=info;

  ftb->keynr=keynr;



  init_alloc_root(&ftb->mem_root, query_len,0);

  init_alloc_root(&ftb->mem_root, 1024, 1024);



  /* hack: instead of init_queue, we'll use reinit queue to be able

   * to alloc queue with alloc_root()
@@ -201,7 +205,7 @@ FTB * ft_boolean_search_init(MI_INFO *info, uint keynr, byte *query, 
  return ftb;

}



int ft_boolean_search_next(FTB *ftb, char *record)

int ft_boolean_read_next(FT_INFO *ftb, char *record)

{

  FTB_EXPR  *ftbe, *up;

  FTB_WORD  *ftbw;
@@ -218,14 +222,16 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
    return my_errno;

  /* black magic OFF */



  while(ftb->ok && ftb->queue.elements)

  {

    curdoc=((FTB_WORD *)queue_top(& ftb->queue))->docid;

  if (!ftb->queue.elements)

    return my_errno=HA_ERR_END_OF_FILE;



  while(ftb->ok &&

    (curdoc=((FTB_WORD *)queue_top(& ftb->queue))->docid) != HA_POS_ERROR)

  {

    while (curdoc==(ftbw=(FTB_WORD *)queue_top(& ftb->queue))->docid)

    {

      float weight=ftbw->weight;

      uint  yn=ftbw->yesno;

      int  yn=ftbw->yesno;

      for (ftbe=ftbw->up; ftbe; ftbe=ftbe->up)

      {

        if (ftbe->docid != curdoc)
@@ -248,9 +254,9 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
        if (yn<0)

        {

         /* NOTE: special sort function of queue assures that all yn<0

	   * events for every particular subexpression will happen

	   * BEFORE all yn>=0 events. So no already matched expression

	   * can become not-matched again.

          * events for every particular subexpression will

          * "auto-magically" happen BEFORE all yn>=0 events. So no

          * already matched expression can become not-matched again.

          */

          ++ftbe->nos;

          break;
@@ -261,6 +267,7 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
          if (ftbe->yesses >= ftbe->ythresh && !ftbe->nos)

          {

            yn=ftbe->yesno;

            ftbe->cur_weight=weight;

            weight*=ftbe->weight;

          }

          else
@@ -272,7 +279,7 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
      }

      /* update queue */

      r=_mi_search(info, keyinfo, ftbw->word, ftbw->len,

                   SEARCH_FIND | SEARCH_PREFIX, keyroot);

                   SEARCH_BIGGER , keyroot);

      if (!r)

      {

        r=_mi_compare_text(default_charset_info,
@@ -281,7 +288,7 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
      }

      if (r) /* not found */

      {

	queue_remove(& ftb->queue, 0);

        ftbw->docid=HA_POS_ERROR;

        if (ftbw->yesno>0 && ftbw->up->up==0)

        { /* this word MUST BE present in every document returned,

             so we can stop the search right now */
@@ -292,8 +299,8 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
      {

        memcpy(ftbw->word, info->lastkey, info->lastkey_length);

        ftbw->docid=info->lastpos;

        queue_replaced(& ftb->queue);

      }

      queue_replaced(& ftb->queue);

    }



    ftbe=ftb->root;
@@ -314,3 +321,30 @@ int ft_boolean_search_next(FTB *ftb, char *record) 
  return my_errno=HA_ERR_END_OF_FILE;

}



float ft_boolean_find_relevance(FT_INFO *ftb, my_off_t docid)

{

  fprintf(stderr, "ft_boolean_find_relevance called!\n");

  return -1.0; /* to be done via str scan */

}



void ft_boolean_close_search(FT_INFO *ftb)

{

  free_root(& ftb->mem_root, MYF(0));

  my_free((gptr)ftb,MYF(0));

}



float ft_boolean_get_relevance(FT_INFO *ftb)

{

  return ftb->root->cur_weight;

}



my_off_t ft_boolean_get_docid(FT_INFO *ftb)

{

  return HA_POS_ERROR;

}



void ft_boolean_reinit_search(FT_INFO *ftb)

{

  fprintf(stderr, "ft_boolean_reinit_search called!\n");

}