naoa
null+****@clear*****
Sun Oct 26 18:27:34 JST 2014
naoa 2014-10-26 18:27:34 +0900 (Sun, 26 Oct 2014) New Revision: 7a28e8167b47c7f3f1fc6325c2bdf44873a52154 https://github.com/groonga/groonga/commit/7a28e8167b47c7f3f1fc6325c2bdf44873a52154 Merged ff11b0c: Merge pull request #232 from naoa/tokenize-support-token_filters Message: tokenize: remove table option Modified files: lib/proc.c test/command/suite/tokenize/with_token_filters.expected test/command/suite/tokenize/with_token_filters.test Modified: lib/proc.c (+5 -12) =================================================================== --- lib/proc.c 2014-10-26 17:24:38 +0900 (15948b6) +++ lib/proc.c 2014-10-26 18:27:34 +0900 (74b85d6) @@ -3545,7 +3545,6 @@ proc_tokenize(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) grn_obj *flag_names; grn_obj *mode_name; grn_obj *token_filter_names; - grn_obj *table_name; tokenizer_name = VAR(0); string = VAR(1); @@ -3553,7 +3552,6 @@ proc_tokenize(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) flag_names = VAR(3); mode_name = VAR(4); token_filter_names = VAR(5); - table_name = VAR(6); if (GRN_TEXT_LEN(tokenizer_name) == 0) { ERR(GRN_INVALID_ARGUMENT, "[tokenize] tokenizer name is missing"); @@ -3574,14 +3572,10 @@ proc_tokenize(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) return NULL; } - if (GRN_TEXT_LEN(table_name)) { - lexicon = grn_ctx_get(ctx, GRN_TEXT_VALUE(table_name), GRN_TEXT_LEN(table_name)); - } else { - lexicon = create_lexicon_for_tokenize(ctx, - tokenizer_name, - normalizer_name, - token_filter_names); - } + lexicon = create_lexicon_for_tokenize(ctx, + tokenizer_name, + normalizer_name, + token_filter_names); if (!lexicon) { return NULL; } @@ -5499,8 +5493,7 @@ grn_db_init_builtin_query(grn_ctx *ctx) DEF_VAR(vars[3], "flags"); DEF_VAR(vars[4], "mode"); DEF_VAR(vars[5], "token_filters"); - DEF_VAR(vars[6], "table"); - DEF_COMMAND("tokenize", proc_tokenize, 7, vars); + DEF_COMMAND("tokenize", proc_tokenize, 6, vars); DEF_COMMAND("tokenizer_list", proc_tokenizer_list, 0, vars); Modified: test/command/suite/tokenize/with_token_filters.expected (+7 -20) =================================================================== --- test/command/suite/tokenize/with_token_filters.expected 2014-10-26 17:24:38 +0900 (87c7367) +++ test/command/suite/tokenize/with_token_filters.expected 2014-10-26 18:27:34 +0900 (b5ce78f) @@ -1,15 +1,6 @@ -register token_filters/stop_word +register token_filters/stem [[0,0.0,0.0],true] -table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto --token_filters TokenFilterStopWord -[[0,0.0,0.0],true] -column_create Terms is_stop_word COLUMN_SCALAR Bool -[[0,0.0,0.0],true] -load --table Terms -[ -{"_key": "and", "is_stop_word": true} -] -[[0,0.0,0.0],1] -tokenize TokenBigram "Hello and Good-bye" --mode GET --table Terms +tokenize TokenBigram "I developed Groonga" NormalizerAuto --token_filters TokenFilterStem [ [ 0, @@ -18,20 +9,16 @@ tokenize TokenBigram "Hello and Good-bye" --mode GET --table Terms ], [ { - "value": "hello", + "value": "i", "position": 0 }, { - "value": "good", - "position": 2 - }, - { - "value": "-", - "position": 3 + "value": "develop", + "position": 1 }, { - "value": "bye", - "position": 4 + "value": "groonga", + "position": 2 } ] ] Modified: test/command/suite/tokenize/with_token_filters.test (+4 -13) =================================================================== --- test/command/suite/tokenize/with_token_filters.test 2014-10-26 17:24:38 +0900 (19a85a5) +++ test/command/suite/tokenize/with_token_filters.test 2014-10-26 18:27:34 +0900 (899e966) @@ -1,14 +1,5 @@ -register token_filters/stop_word +#@on-error omit +register token_filters/stem +#@on-error default -table_create Terms TABLE_PAT_KEY ShortText \ - --default_tokenizer TokenBigram \ - --normalizer NormalizerAuto \ - --token_filters TokenFilterStopWord -column_create Terms is_stop_word COLUMN_SCALAR Bool - -load --table Terms -[ -{"_key": "and", "is_stop_word": true} -] - -tokenize TokenBigram "Hello and Good-bye" --mode GET --table Terms +tokenize TokenBigram "I developed Groonga" NormalizerAuto --token_filters TokenFilterStem -------------- next part -------------- HTML����������������������������...Download
