// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.tokenize;

import java.util.Arrays;
import org.lemurproject.galago.core.parse.Document;
import org.lemurproject.galago.tupleflow.FakeParameters;
import org.lemurproject.galago.utility.Parameters;
import org.lemurproject.galago.tupleflow.TupleFlowParameters;

/**
 *
 * @author jfoley
 */
public class NewlineTokenizer extends Tokenizer {
  
  public NewlineTokenizer(TupleFlowParameters tfp) {
    super(tfp);
  }
  public NewlineTokenizer() {
    super(new FakeParameters(Parameters.create()));
  }

  @Override
  public void tokenize(Document input) {
    input.terms = Arrays.asList(input.text.split("\n"));
  }
}

@@ -101,33 +101,8 @@
 e;%0A%0A
-import java.util.Arrays;%0A
 impo
@@ -241,16 +241,27 @@
 ago.
-utility.
+tupleflow.TupleFlow
 Para
@@ -303,36 +303,81 @@
 ago.
-tupleflow.TupleFlowParameter
+utility.Parameters;%0A%0Aimport java.util.Arrays;%0Aimport java.util.Collection
 s;%0A%0A
@@ -724,18 +724,60 @@
 lit(%22%5Cn%22));%0A
+    input.tags = Collections.emptyList();%0A
   %7D%0A%7D%0A

Fix the fact that NewlineTokenizer wasn't outputting good documents (needs a tags list)