// $Id$


import java.io.*;
import java.util.*;

public class ProcessShallowParse
{
	public static void main(String[] args) throws Exception
	{
		System.err.println("Starting...");

		InputStreamReader inStream = args.length > 0 ? new FileReader(args[0]) : new InputStreamReader(System.in); 
		OutputStreamWriter outStream = args.length > 1 ?  new FileWriter(args[1]) : new OutputStreamWriter(System.out); 
		
		new ProcessShallowParse(inStream, outStream);
		
		System.err.println("End...");
	}

	public ProcessShallowParse(Reader inStream, Writer outStream) throws Exception
	{
		BufferedReader inFile = new BufferedReader(inStream); 
		BufferedWriter outFile = new BufferedWriter(outStream); 
		
		// tokenise
		String inLine;
		while ((inLine = inFile.readLine()) != null)
		{
			StringTokenizer st = new StringTokenizer(inLine);
		     while (st.hasMoreTokens()) 
		     {
		    	 String token = st.nextToken();
		    	 if (token.substring(0, 2).compareTo("I-") != 0)
		    		 outFile.write(token + " ");
		     }
		     outFile.write("\n");
		}		
	}
}


@@ -428,16 +428,17 @@
 lowParse
+2
 (inStrea
@@ -1053,8 +1053,1101 @@
 %09%0A%09%7D%0A%7D%0A%0A
+class ProcessShallowParse2%0A%7B // factored sentence%0A%09public ProcessShallowParse2(Reader inStream, Writer outStream) throws Exception%0A%09%7B%09%09%0A%09%09BufferedReader inFile = new BufferedReader(inStream); %0A%09%09BufferedWriter outFile = new BufferedWriter(outStream); %0A%09%09%0A%09%09// tokenise%0A%09%09String inLine;%0A%09%09while ((inLine = inFile.readLine()) != null)%0A%09%09%7B%0A%09%09%09StringTokenizer st = new StringTokenizer(inLine);%0A%09%09     while (st.hasMoreTokens()) %0A%09%09     %7B%0A%09%09    %09 String factoredWord = st.nextToken();%0A%09%09    %09 Output(factoredWord, outFile);%0A%09%09     %7D%0A%09%09     outFile.write(%22%5Cn%22);%0A%09%09%7D%09%09%0A%09%7D%0A%09%0A%09protected void Output(String factoredWord, BufferedWriter outStream) throws Exception%0A%09%7B%0A%09%09StringTokenizer st = new StringTokenizer(factoredWord, %22%7C%22);%0A    %09st.nextToken();%0A    %09String pos = st.nextToken();%0A    %09String morph = st.nextToken();%0A%0A    %09int lastPos = pos.lastIndexOf('-');%0A    %09%0A    %09if (pos.indexOf(%22ART%22) == 0%0A    %09%09%7C%7C pos.indexOf(%22P%22) == 0%0A    %09%09%7C%7C pos.indexOf(%22V%22) == 0%0A    %09%09%7C%7C pos.indexOf(%22$,%22) == 0%0A    %09%09%7C%7C pos.indexOf(%22$.%22) == 0%0A    %09%09)%0A    %09%7B%0A    %09%09outStream.write(pos + %22%7C%22 + morph + %22 %22);%0A    %09%7D%0A%09%7D%0A%7D%0A

create data for chunking LM