Groonga - fulltext search engine.

[Groonga-commit] groonga/gcs [master] Return no result if the query includes unknown field

YUKI Hiroshi null+****@clear*****
Fri Aug 24 19:24:55 JST 2012

• Previous message: [Groonga-commit] [Still Failing] mroonga/mroonga#25 (master - 4204656)
• Next message: [Groonga-commit] groonga/gcs [master] Split SelectQuery implementation
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

YUKI Hiroshi	2012-08-24 19:24:55 +0900 (Fri, 24 Aug 2012)

  New Revision: f71d3f8818c5550bd3e0b16044846db8653eff27
  https://github.com/groonga/gcs/commit/f71d3f8818c5550bd3e0b16044846db8653eff27

  Log:
    Return no result if the query includes unknown field

  Modified files:
    lib/api/2011-02-01/search.js
    lib/bq-translator.js
    test/bq-translator.test.js

  Modified: lib/api/2011-02-01/search.js (+162 -134)
===================================================================
--- lib/api/2011-02-01/search.js    2012-08-24 18:24:30 +0900 (d4f10cd)
+++ lib/api/2011-02-01/search.js    2012-08-24 19:24:55 +0900 (ee38d50)
@@ -4,6 +4,8 @@ var Domain = require('../../database').Domain;
 var nroonga = require('../../wrapped-nroonga');
 var BooleanQueryTranslator = require('../../bq-translator').BooleanQueryTranslator;
 
+var dummyRid = '000000000000000000000000000000000000000000000000000000000000000';
+
 function formatFacets(data) {
   var drilldownRecords = data.slice(1);
 
@@ -63,8 +65,12 @@ function formatSelectResults(data) {
   return results;
 }
 
-function select(context, options, callback) {
-  context.command('select', options, function(error, data) {
+function select(context, selectQuery, callback) {
+  if (selectQuery.noResult) {
+    callback(null, [], 0, []);
+    return;
+  }
+  context.command('select', selectQuery.selectOptions, function(error, data) {
     if (error) {
       callback(error);
     } else {
@@ -79,6 +85,24 @@ function select(context, options, callback) {
   });
 }
 
+function createResultBody(options) {
+  var info = {
+    rid:           dummyRid,
+    'time-ms':     options.elapsedTime || 0,
+    'cpu-time-ms': 0 // TODO
+  };
+  return {
+    rank: '-text_relevance', // FIXME
+    'match-expr': options.matchExpression,
+    hits: {
+      found: options.found,
+      start: options.start,
+      hit:   options.hit
+    },
+    info: info
+  };
+}
+
 function createErrorBody(options) {
   var messages = [];
   if (options.messages) {
@@ -86,16 +110,16 @@ function createErrorBody(options) {
   } else {
     messages.push({
       severity: 'fatal',
-      code: '',
-      message: options.message || ''
+      code:     '',
+      message:  options.message || ''
     });
   }
   return {
-    error: 'info',
-    rid: options.rid,
-    'time-ms': options.elapsedTime || 0,
+    error:         'info',
+    rid:           options.rid,
+    'time-ms':     options.elapsedTime || 0,
     'cpu-time-ms': 0, // TODO
-    messages: messages
+    messages:      messages
   };
 }
 
@@ -103,152 +127,156 @@ function translateQueryToBooleanQuery(query) {
   return "'" + query.replace(/(['\\])/g, '\\$1') + "'";
 }
 
-var dummyRid = '000000000000000000000000000000000000000000000000000000000000000';
+function SelectQuery(request, context) {
+  var domain = new Domain(request, context);
+  var query = request.query.q || '';
+  var booleanQuery = request.query.bq || '';
+  var filters = [];
+  var matchExpression = '';
+  var facets = request.query.facet;
+  var noResult = false;
 
-exports.createHandler = function(context) {
-  return function(request, response) {
-    var startedAt = new Date();
-    var domain = new Domain(request, context);
-    var query = request.query.q || '';
-    var booleanQuery = request.query.bq || '';
-    var filters = [];
-    var matchExpr = '';
-    var facetParameter = request.query.facet;
-
-    var defaultFields;
-    var defaultField = domain.defaultSearchField;
-    if (defaultField)
-      defaultFields = [defaultField];
-    else
-      defaultFields = domain.searchableIndexFields.filter(function(field) {
-        return field.type == 'text';
-      });
-
-    var defaultFieldNames = defaultFields.map(function(field) {
-      return field.name;
+  var defaultFields;
+  var defaultField = domain.defaultSearchField;
+  if (defaultField)
+    defaultFields = [defaultField];
+  else
+    defaultFields = domain.searchableIndexFields.filter(function(field) {
+      return field.type == 'text';
     });
 
-    if (query) {
-      var queryAsBooleanQuery = translateQueryToBooleanQuery(query);
-      var translator = new BooleanQueryTranslator(queryAsBooleanQuery);
-      translator.domain = domain;
-      translator.defaultFieldNames = defaultFieldNames;
-      try {
-        filters.push(translator.translate());
-      } catch (error) {
-        var body = createErrorBody({
-          rid: dummyRid,
-          message: 'Invalid q value: ' + (error.message || error)
-        });
-        return response.send(body, 400);
-      }
-      matchExpr = '(label ' + queryAsBooleanQuery + ')';
-    }
+  var defaultFieldNames = defaultFields.map(function(field) {
+    return field.name;
+  });
 
-    if (booleanQuery) {
-      var translator = new BooleanQueryTranslator(booleanQuery);
-      translator.domain = domain;
-      translator.defaultFieldNames = defaultFieldNames;
-      try {
-        filters.push(translator.translate());
-      } catch (error) {
-        var errorData = { rid: dummyRid };
-        if (error.message == 'validation error') {
-          errorData.messages = [error.data];
-        } else if (error.message == 'unsearchable field') {
-          errorData.message = 'unsearchable field '+error.fieldName;
-        } else {
-          errorData.message = 'Invalid bq value: ' + (error.message || error);
-        }
-        var body = createErrorBody(errorData);
-        return response.send(body, 400);
-      }
-      if (matchExpr.length > 0) {
-        matchExpr = '(and ' + matchExpr + ' ' + booleanQuery + ')';
-      } else {
-        matchExpr = booleanQuery;
-      }
+  if (query) {
+    var queryAsBooleanQuery = translateQueryToBooleanQuery(query);
+    var translator = new BooleanQueryTranslator(queryAsBooleanQuery);
+    translator.domain = domain;
+    translator.defaultFieldNames = defaultFieldNames;
+    try {
+      filters.push(translator.translate());
+    } catch (error) {
+      error.queryType = 'q';
+      throw error;
     }
+    matchExpression = '(label ' + queryAsBooleanQuery + ')';
+  }
 
-    filters = filters.map(function(filter) {
-      return '(' + filter + ')';
-    });
-    var size = parseInt(request.query.size || '10', 10);
-    var start = parseInt(request.query.start || '0', 10);
-    var filter = filters.join(' && ');
-    var requestedOutputColumns = request.query['return-fields'] || '';
-    requestedOutputColumns = requestedOutputColumns.split(/\s*,\s*/);
-    var outputColumns = domain.resultReturnableIndexFields
-                              .filter(function(field) {
-                                return requestedOutputColumns.indexOf(field.name) > -1;
-                              })
-                              .map(function(field) {
-                                return field.columnName;
-                              });
-    outputColumns.unshift('_key');
-    var options = {
-      table:          domain.tableName,
-      filter:         filter,
-      limit:          size,
-      offset:         start,
-      output_columns: outputColumns.join(', ')
-    };
-
-    if (domain.hasSynonymsTableSync()) {
-      options.query_expansion = domain.synonymsTableName + '.synonyms';
+  if (booleanQuery) {
+    var translator = new BooleanQueryTranslator(booleanQuery);
+    translator.domain = domain;
+    translator.defaultFieldNames = defaultFieldNames;
+    try {
+      filters.push(translator.translate());
+    } catch (error) {
+      error.queryType = 'bq';
+      throw error;
     }
-    if (filter) {
-      options.filter = filter;
+    noResult = noResult || !translator.available;
+    if (matchExpression.length > 0) {
+      matchExpression = '(and ' + matchExpression + ' ' + booleanQuery + ')';
+    } else {
+      matchExpression = booleanQuery;
     }
+  }
+
+  filters = filters.map(function(filter) {
+    return '(' + filter + ')';
+  });
+  var size = parseInt(request.query.size || '10', 10);
+  var start = parseInt(request.query.start || '0', 10);
+  var filter = filters.join(' && ');
+  var requestedOutputColumns = request.query['return-fields'] || '';
+  requestedOutputColumns = requestedOutputColumns.split(/\s*,\s*/);
+  var outputColumns = domain.resultReturnableIndexFields
+                            .filter(function(field) {
+                              return requestedOutputColumns.indexOf(field.name) > -1;
+                            })
+                            .map(function(field) {
+                              return field.columnName;
+                            });
+  outputColumns.unshift('_key');
+  var options = {
+    table:          domain.tableName,
+    filter:         filter,
+    limit:          size,
+    offset:         start,
+    output_columns: outputColumns.join(', ')
+  };
+
+  if (domain.hasSynonymsTableSync()) {
+    options.query_expansion = domain.synonymsTableName + '.synonyms';
+  }
+  if (filter) {
+    options.filter = filter;
+  }
+
+  if (facets) {
+    var facetReturnableFields = domain.facetReturnableIndexFields
+                                  .map(function(field) {
+                                    return field.name;
+                                  });
+    facets = facets.split(/\s*,\s*/)
+               .filter(function(field) {
+                 return facetReturnableFields.indexOf(field) > -1;
+               });
+    options.drilldown = facets.join(',');
+    options.drilldown_sortby = '-_nsubrecs';
+    // TODO support sorting parameter
+    // TODO support facet-FIELD-top-n parameter
+  }
 
-    if (facetParameter) {
-      var facetReturnableFields = domain.facetReturnableIndexFields
-                                    .map(function(field) {
-                                      return field.name;
-                                    });
-      facetParameter = facetParameter.split(/\s*,\s*/)
-                         .filter(function(field) {
-                           return facetReturnableFields.indexOf(field) > -1;
-                         })
-                         .join(',');
-      options.drilldown = facetParameter;
-      options.drilldown_sortby = '-_nsubrecs';
-      // TODO support sorting parameter
-      // TODO support facet-FIELD-top-n parameter
+  return {
+    selectOptions:   options,
+    matchExpression: matchExpression,
+    start:           start,
+    facets:          facets,
+    noResult:        noResult
+  };
+}
+
+exports.createHandler = function(context) {
+  return function(request, response) {
+    var startedAt = new Date();
+
+    var selectQuery;
+    try {
+      selectQuery = new SelectQuery(request, context);
+    } catch(error) {
+      var errorData = { rid: dummyRid };
+      if (error.message == 'validation error') {
+        errorData.messages = [error.data];
+      } else {
+        errorData.message = 'Invalid ' + error.queryType + ' value: ' + (error.message || error) + error.stack;
+      }
+      var body = createErrorBody(errorData);
+      return response.send(body, 400);
     }
 
-    select(context, options,
+    select(context, selectQuery,
       function(error, data, numFoundRecords, facets) {
         var finishedAt = new Date();
-        var elapsedTime = finishedAt - startedAt;
-        var info = {
-          rid: dummyRid,
-          'time-ms': elapsedTime,
-          'cpu-time-ms': 0 // TODO
-        };
+        var elapsedTime = finishedAt.getTime() - startedAt.getTime();
         if (error) {
           var body = createErrorBody({
-            rid: dummyRid,
-            message: error.message,
+            rid:         dummyRid,
+            message:     error.message,
             elapsedTime: elapsedTime
           });
           return response.send(body, 400); // TODO
         }
-        var result = {
-          rank: '-text_relevance', // FIXME
-          'match-expr': matchExpr,
-          hits: {
-            found: numFoundRecords,
-            start: start,
-            hit: data
-          },
-          info: info
-        };
-        if (facetParameter) {
-          var facetNames = facetParameter.split(',');
+        var result = createResultBody({
+          matchExpression: selectQuery.matchExpression,
+          found:           numFoundRecords,
+          start:           selectQuery.start,
+          hit:             data,
+          elapsedTime:     elapsedTime
+        });
+        if (selectQuery.facets && selectQuery.facets.length) {
           var facetsObject = {};
-          facets.map(function(facet, index) {
-            var facetName = facetNames[index];
+          facets.forEach(function(facet, index) {
+            var facetName = selectQuery.facets[index];
             facetsObject[facetName] = facet;
           });
           result.facets = facetsObject;

  Modified: lib/bq-translator.js (+3 -7)
===================================================================
--- lib/bq-translator.js    2012-08-24 18:24:30 +0900 (4f23a84)
+++ lib/bq-translator.js    2012-08-24 19:24:55 +0900 (ebeb402)
@@ -20,6 +20,7 @@ function BooleanQueryTranslator(query) {
   this.offset = 0;
   this.domain = null;
   this.defaultFieldNames = null;
+  this.available = true;
 }
 
 BooleanQueryTranslator.prototype = {
@@ -56,7 +57,7 @@ BooleanQueryTranslator.prototype = {
     this.throwValidationError(
       'CS-InvalidMatchSetExpression',
       '[WARNING] Syntax error in match set expression: ' +
-        message + ' ' + detial
+        message + ' ' + detail
     );
   },
   throwValidationError: function(code, message) {
@@ -68,11 +69,6 @@ BooleanQueryTranslator.prototype = {
     };
     throw error;
   },
-  throwUnsearchableError: function(fieldName) {
-    var error = new Error('unsearchable field');
-    error.fieldName = fieldName;
-    throw error;
-  },
   skipSpaces: function() {
     for (; this.offset < this.query.length; this.offset++) {
       if (this.query[this.offset] != " ") {
@@ -439,7 +435,7 @@ BooleanQueryTranslator.prototype = {
             'defined in the metadata.'
         );
       } else if (!field.searchEnabled) {
-        this.throwUnsearchableError(fieldName);
+        this.available = false;
       }
     }
     if (!field)

  Modified: test/bq-translator.test.js (+16 -8)
===================================================================
--- test/bq-translator.test.js    2012-08-24 18:24:30 +0900 (14883ba)
+++ test/bq-translator.test.js    2012-08-24 19:24:55 +0900 (9871c7f)
@@ -23,6 +23,14 @@ function testQuery(label, query, expected, customSetup) {
   });
 }
 
+function syntaxErrorData(message) {
+  return {
+    severity: 'fatal',
+    code:     'CS-InvalidMatchSetExpression',
+    message:  '[WARNING] Syntax error in match set expression: ' + message
+  };
+}
+
 function testQueryError(label, query, context, detail) {
   test('error: query: ' + label + ': ' + '<' + query + '>', function() {
     var translator = createTranslator(query);
@@ -35,7 +43,8 @@ function testQueryError(label, query, context, detail) {
         throw error;
       }
     });
-    assert.equal(actualError.message, "<" + context + ">" + ": " + detail);
+    assert.deepEqual(actualError.data,
+                     syntaxErrorData(detail + ' <' + context + '>'));
   });
 }
 
@@ -67,7 +76,8 @@ function testGroupError(label, group, context, detail) {
         throw error;
       }
     });
-    assert.equal(actualError.message, "<" + context + ">" + ": " + detail);
+    assert.deepEqual(actualError.data,
+                     syntaxErrorData(detail + ' <' + context + '>'));
   });
 }
 
@@ -102,7 +112,8 @@ function testExpressionError(label, expression, context, detail) {
         throw error;
       }
     });
-    assert.equal(actualError.message, "<" + context + ">" + ": " + detail);
+    assert.deepEqual(actualError.data,
+                     syntaxErrorData(detail + ' <' + context + '>'));
   });
 }
 
@@ -116,8 +127,7 @@ function testDefaultFieldNames(label, query, defaultFieldNames, expected) {
   });
 }
 
-function testDefaultFieldNamesError(label, query, defaultFieldNames,
-                                    context, detail) {
+function testDefaultFieldNamesError(label, query, defaultFieldNames, detail) {
   test('error: default field names: ' + label + ': ' + '<' + query + '>',
        function() {
     var translator = createTranslator(query);
@@ -131,7 +141,7 @@ function testDefaultFieldNamesError(label, query, defaultFieldNames,
         throw error;
       }
     });
-    assert.equal(actualError.message, "<" + context + ">" + ": " + detail);
+    assert.equal(actualError.data, detail);
   });
 }
 
@@ -388,12 +398,10 @@ suite('BoolanQueryTranslator', function() {
   testDefaultFieldNamesError("null",
                              "'ModelName'",
                              null,
-                             "'ModelName'||",
                              "default fields are missing");
   testDefaultFieldNamesError("empty",
                              "'ModelName'",
                              [],
-                             "'ModelName'||",
                              "no default field");
 
   testSynonym("keyword: existent: 0 synonym",
-------------- next part --------------
HTML����������������������������...
Download 

• Previous message: [Groonga-commit] [Still Failing] mroonga/mroonga#25 (master - 4204656)
• Next message: [Groonga-commit] groonga/gcs [master] Split SelectQuery implementation
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]