/*
 * Copyright 2000-2005 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.lang.cacheBuilder;

import com.intellij.util.Processor;

/**
 * The default primitive implementation of a words scanner. The implementation does not
 * use any lexer, breaks text into words at boundaries of sequences of English letters
 * and treats all words as occurred in unknown context.
 *
 * @author max
 */
public class SimpleWordsScanner implements WordsScanner {
  public void processWords(CharSequence fileText, Processor<WordOccurrence> processor) {
    int index = 0;
    WordOccurrence occurrence = null;

    ScanWordsLoop:
    while (true) {
      while (true) {
        if (index == fileText.length()) break ScanWordsLoop;
        char c = fileText.charAt(index);
        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
            (Character.isJavaIdentifierStart(c) && c != '$')) {
          break;
        }
        index++;
      }
      int index1 = index;
      while (true) {
        index++;
        if (index == fileText.length()) break;
        char c = fileText.charAt(index);
        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) continue;
        if (!Character.isJavaIdentifierPart(c) || c == '$') break;
      }
      if (index - index1 > 100) continue; // Strange limit but we should have some!

      if (occurrence == null) occurrence = new WordOccurrence(fileText, index1, index, null);
      else occurrence.init(fileText, index1, index, null);
      processor.process(occurrence);
    }
  }
}

@@ -670,16 +670,61 @@
 ocessor;
+%0Aimport com.intellij.util.text.CharArrayUtil;
 %0A%0A/**%0A *
@@ -1177,16 +1177,101 @@
  = null;
+%0A    final char%5B%5D fileTextArray = CharArrayUtil.fromSequenceWithoutCopying(fileText);
 %0A%0A    Sc
@@ -1384,32 +1384,38 @@
 dsLoop;%0A        
+final 
 char c = fileTex
@@ -1399,32 +1399,77 @@
  final char c = 
+fileTextArray != null ? fileTextArray%5Bindex%5D:
 fileText.charAt(
@@ -1801,16 +1801,22 @@
         
+final 
 char c =
@@ -1808,32 +1808,77 @@
  final char c = 
+fileTextArray != null ? fileTextArray%5Bindex%5D:
 fileText.charAt(

Refactor Lexer to use CharSequence instead of char[] (IDEADEV-12089)