/**
 * 
 */
package codemining.lm.ngram.tui;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
import java.util.logging.Logger;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;

import codemining.languagetools.ITokenizer;
import codemining.lm.ngram.AbstractNGramLM;
import codemining.lm.ngram.NGramLM;
import codemining.util.serialization.ISerializationStrategy.SerializationException;

/**
 * @author Miltos Allamanis <m.allamanis@ed.ac.uk>
 * 
 */
public class NGramModelBuilder {

	private static final Logger LOGGER = Logger
			.getLogger(NGramModelBuilder.class.getName());

	public static void main(final String[] args) throws IOException,
			ClassNotFoundException, InstantiationException,
			IllegalAccessException, IllegalArgumentException,
			InvocationTargetException, NoSuchMethodException,
			SecurityException, SerializationException {

		if (args.length != 5) {
			System.err
					.println("Usage <TrainingFiles> <N> <NGramModel.ser output> <tokenizationClass> <WrapperSmootherClass>");
			return;
		}

		final Class<? extends ITokenizer> tokenizerName = (Class<? extends ITokenizer>) Class
				.forName(args[3]);
		final ITokenizer tokenizer = tokenizerName.newInstance();

		final NGramLM dict = new NGramLM(Integer.parseInt(args[1]), tokenizer);

		LOGGER.info("NGram Model creater started with " + args[1]
				+ "-gram for files in " + args[0] + " using " + args[3]
				+ " tokenizer");

		final Collection<File> files = FileUtils.listFiles(new File(args[0]),
				dict.modelledFilesFilter(), DirectoryFileFilter.DIRECTORY);

		dict.trainModel(files);

		LOGGER.info("Ngram model build. Adding Smoother...");
		final Class<? extends AbstractNGramLM> smoothedNgramClass = (Class<? extends AbstractNGramLM>) Class
				.forName(args[4]);
		final AbstractNGramLM ng = (AbstractNGramLM) smoothedNgramClass
				.getDeclaredConstructor(AbstractNGramLM.class)
				.newInstance(dict);

		LOGGER.info("Ngram model build. Serializing...");
		ng.serializeToDisk(args[2]);

	}
}

@@ -349,16 +349,64 @@
 enizer;%0A
+import codemining.languagetools.TokenizerUtils;%0A
 import c
@@ -1201,32 +1201,16 @@
 %09%09final 
-Class%3C? extends 
 ITokeniz
@@ -1211,17 +1211,16 @@
 okenizer
-%3E
  tokeniz
@@ -1225,131 +1225,51 @@
 izer
-Name = (Class%3C? extends ITokenizer%3E) Class%0A%09%09%09%09.forName(args%5B3%5D);%0A%09%09final ITokenizer tokenizer = tokenizerName.newInstance(
+ = TokenizerUtils.tokenizerForClass(args%5B3%5D
 );%0A%0A

Refactor tokenizer from class name code into TokenizerUtils