var _ = require('lodash');
var log = require('winston');
var walk = require('../../../lib/utils/walk');
var fs = require('fs');
var path = require('canonical-path');

// Keywords to ignore
var wordsToIgnore = [];

// Keywords start with "ng:" or one of $, _ or a letter
var KEYWORD_REGEX = /^((ng:|[\$_a-z])[\w\-_]+)/;

module.exports = {
  name: 'keywords',
  runAfter: ['docs-processed'],
  runBefore: ['adding-extra-docs'],
  description: 'This processor extracts all the keywords from the document',
  init: function(config) {

    // Load up the keywords to ignore, if specified in the config
    if ( config.processing.search && config.processing.search.ignoreWordsFile ) {

      var ignoreWordsPath = path.resolve(config.basePath, config.processing.search.ignoreWordsFile);
      wordsToIgnore = fs.readFileSync(ignoreWordsPath, 'utf8').toString().split(/[,\s\n\r]+/gm);

    }
  },
  process: function(docs) {

    // keywordMap holds a map of words that have been found already
    var keywordMap = _.indexBy(wordsToIgnore);
    var words = [];

    // If the title contains a name starting with ng, e.g. "ngController", then add the module name
    // without the ng to the title text, e.g. "controller".
    function extractTitleWords(title) {
      var match = /ng([A-Z]\w*)/.exec(title);
      if ( match ) {
        title = title + ' ' + match[1].toLowerCase();
      }
      return title;
    }

    function extractWords(text) {
      if (_.isString(text)) {
        var tokens = text.toLowerCase().split(/[\.\s,`'"#]+/mg);
        _.forEach(tokens, function(token){
          var match = token.match(KEYWORD_REGEX);
          if (match){
            key = match[1];
            if ( !keywordMap[key]) {
              keywordMap[key] = true;
              words.push(key);
            }
          }
        });
      }
      return text;
    }

    _.forEach(docs, function(doc) {

      _.forEach(doc, extractWords);

      doc.searchTerms = {
        titleWords: extractTitleWords(doc.name),
        keywords: _.sortBy(words).join(' ')
      };
    });
  }
};
@@ -205,16 +205,59 @@
 re = %5B%5D;
+%0Avar propertiesToIgnore;%0Avar areasToSearch;
 %0A%0A// Key
@@ -924,111 +924,426 @@
     
-%7D%0A  %7D,%0A  process: function(docs) %7B%0A%0A    // keywordMap holds a map of words that have been found already
+  log.debug('Loaded ignore words from %22' + ignoreWordsPath + '%22');%0A      log.silly(wordsToIgnore);%0A%0A    %7D%0A%0A    areasToSearch = _.indexBy(config.get('processing.search.areasToSearch', %5B'api', 'guide', 'misc', 'error', 'tutorial'%5D));%0A%0A    propertiesToIgnore = _.indexBy(config.get('processing.search.propertiesToIgnore', %5B%5D));%0A    log.debug('Properties to ignore', propertiesToIgnore);%0A%0A  %7D,%0A  process: function(docs) %7B%0A
 %0A   
@@ -1347,23 +1347,27 @@
     var 
-keyw
+ignoreW
 ord
+s
 Map = _.
@@ -1393,28 +1393,8 @@
 re);
-%0A    var words = %5B%5D;
 %0A%0A  
@@ -1781,44 +1781,32 @@
 text
-) %7B%0A      if (_.isString(text)
+, words, keywordMap
 ) %7B%0A
-  
+%0A
     
@@ -1864,18 +1864,16 @@
 %5D+/mg);%0A
-  
       _.
@@ -1909,26 +1909,24 @@
 n)%7B%0A        
-  
 var match = 
@@ -1957,26 +1957,24 @@
 X);%0A        
- 
- 
 if (match)%7B%0A
@@ -1973,18 +1973,16 @@
 match)%7B%0A
-  
         
@@ -2005,26 +2005,24 @@
 ;%0A          
- 
- 
 if ( !keywor
@@ -2042,26 +2042,24 @@
             
-  
 keywordMap%5Bk
@@ -2078,26 +2078,24 @@
             
- 
- 
 words.push(k
@@ -2105,30 +2105,26 @@
 ;%0A          
-  %7D%0A  
+%7D%0A
         %7D%0A  
@@ -2117,26 +2117,24 @@
 %7D%0A        %7D%0A
-  
       %7D);%0A  
@@ -2139,108 +2139,512 @@
     
-  %7D
+%7D%0A%0A
 %0A    
-  return text;%0A    %7D%0A%0A    _.forEach(docs, function(doc) %7B%0A%0A      _.forEach(doc, extractWords
+// We are only interested in docs that live in the right area%0A    docs = _.filter(docs, function(doc) %7B return areasToSearch%5Bdoc.area%5D; %7D);%0A%0A    _.forEach(docs, function(doc) %7B%0A%0A      var words = %5B%5D;%0A      var keywordMap = _.clone(ignoreWordsMap);%0A%0A      // Search each top level property of the document for search terms%0A      _.forEach(doc, function(value, key) %7B%0A        if ( _.isString(value) && !propertiesToIgnore%5Bkey%5D ) %7B%0A          extractWords(value, words, keywordMap);%0A        %7D%0A      %7D
 );%0A%0A
@@ -2771,22 +2771,23 @@
      %7D;%0A
+%0A
     %7D);%0A
   %7D%0A%7D;
@@ -2780,14 +2780,15 @@
     %7D);%0A
+%0A
   %7D%0A%7D;

fix(keywords processor): parse the document more effectively