import re
import os
import nltk.data
from nltk.corpus import stopwords

def readData(filename,lang='spanish'):
	'''
	Arguments: Name of file, language (default: Spanish)

	Returns: List of List of sentence words, unfiltered sentence list and
	length of doc

	Returns the [ s1[w1,w2,w3..], s2[w1,w2,w3..] ...], [s1,s2,...]
	and length of doc.
	'''

	sentWordList = []
	sentList = []    
	lenDoc = 0
	with open(filename,'r') as fp:
		for line in fp:
			line = line.strip()
			sentWordList, sentList, lenDoc = cleanLine(line,sentWordList,sentList,lenDoc,lang)

	return sentWordList,sentList,lenDoc


def cleanLine(line,finalList,sentList,lenDoc,lang):
	'''
	Arguments: each line, existing sentence word list, existing unfiltered sentence
	list, length of the document and language.

	Returns: appended sentence word list, appended unfiltered sentence list, and
	length of the document seen so far.

	Performs preprocessing task. Removes stopwords, punctuations and numeric values
	'''

	try:
		tokenizer = nltk.data.load('tokenizers/punkt/'+ lang + '.pickle')
	except:
		print "Language not Supported"
		return None

	tempList = tokenizer.tokenize(line.decode('utf-8'))
	for sent in tempList:
		sentList.append(sent)

	line = line.decode('utf-8').lower()
	notNum = re.compile(r'[0-9]+')
	line = notNum.sub('',line)
	words = line.split(' ')
	impWords = filter(lambda x: x not in stopwords.words(lang), words)
	line = ' '.join(impWords)

	sentWordList = tokenizer.tokenize(line)

	for sent in sentWordList:
		words = re.findall(r'\w+', sent,flags = re.UNICODE | re.LOCALE)
		finalList.append(words)
		lenDoc += len(words)

	return finalList, sentList, lenDoc

@@ -378,12 +378,8 @@
 = %5B%5D
-    
 %0A%09le
@@ -1273,16 +1273,80 @@
 0-9%5D+')%0A
+%09decimal = re.compile(r'%5Cd+.%5Cd+')%0A%0A%09line = decimal.sub('',line)%0A
 %09line = 

solved the sentence tokenization issue