naoa
null+****@clear*****
Fri Aug 15 14:45:38 JST 2014
naoa 2014-08-15 14:45:38 +0900 (Fri, 15 Aug 2014) New Revision: 1a34e4a974034023d7b255fe88c7cc23e8461b1c https://github.com/groonga/groonga/commit/1a34e4a974034023d7b255fe88c7cc23e8461b1c Merged f87c462: Merge pull request #193 from naoa/add-token-position-skip Message: token: add tokenizer token skip with position Modified files: include/groonga/tokenizer.h lib/token.c Modified: include/groonga/tokenizer.h (+8 -6) =================================================================== --- include/groonga/tokenizer.h 2014-08-15 08:59:00 +0900 (4107eb9) +++ include/groonga/tokenizer.h 2014-08-15 14:45:38 +0900 (a35d55c) @@ -152,17 +152,19 @@ GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token typedef unsigned int grn_tokenizer_status; /* GRN_TOKENIZER_TOKEN_CONTINUE means that the next token is not the last one. */ -#define GRN_TOKENIZER_TOKEN_CONTINUE (0) +#define GRN_TOKENIZER_TOKEN_CONTINUE (0) /* GRN_TOKENIZER_TOKEN_LAST means that the next token is the last one. */ -#define GRN_TOKENIZER_TOKEN_LAST (0x01L<<0) +#define GRN_TOKENIZER_TOKEN_LAST (0x01L<<0) /* GRN_TOKENIZER_TOKEN_OVERLAP means that ... */ -#define GRN_TOKENIZER_TOKEN_OVERLAP (0x01L<<1) +#define GRN_TOKENIZER_TOKEN_OVERLAP (0x01L<<1) /* GRN_TOKENIZER_TOKEN_UNMATURED means that ... */ -#define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2) +#define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2) /* GRN_TOKENIZER_TOKEN_REACH_END means that ... */ -#define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3) +#define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3) /* GRN_TOKENIZER_TOKEN_SKIP means that the token is skipped */ -#define GRN_TOKENIZER_TOKEN_SKIP (0x01L<<4) +#define GRN_TOKENIZER_TOKEN_SKIP (0x01L<<4) +/* GRN_TOKENIZER_TOKEN_SKIP means that the token and postion is skipped */ +#define GRN_TOKENIZER_TOKEN_SKIP_WITH_POSITION (0x01L<<5) /* * GRN_TOKENIZER_CONTINUE and GRN_TOKENIZER_LAST are deprecated. They Modified: lib/token.c (+2 -0) =================================================================== --- lib/token.c 2014-08-15 08:59:00 +0900 (54a1f0c) +++ lib/token.c 2014-08-15 14:45:38 +0900 (75ba6ac) @@ -582,6 +582,8 @@ grn_token_next(grn_ctx *ctx, grn_token *token) if (status & GRN_TOKENIZER_TOKEN_SKIP) { token->pos++; continue; + } else if (status & GRN_TOKENIZER_TOKEN_SKIP_WITH_POSITION) { + continue; } if (token->curr_size == 0) { char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE]; -------------- next part -------------- HTML����������������������������...Download
