# -*- coding: utf-8 -*-

import spacy
import kindred
from intervaltree import IntervalTree
from collections import defaultdict

class Parser:
	"""
	Runs CoreNLP on corpus to get sentences and associated tokens
	"""
	
	def __init__(self,corenlp_url=None,useConstituencyParserOnly=False,language='english'):
		"""
		Create a Parser object that will use CoreNLP for parsing
		
		:param corenlp_url: URL of the CoreNLP instance
		:param useConstituencyParserOnly: Use CoreNLP's constituency parser (with a conversion) for the depenency parse information, and not the CoreNLP dependency parser. This is slower
		:param language: Language to parse (english/arabic/chinese/french/german/spanish). The parser will check that the current CoreNLP is using the matching language. If no CoreNLP instance is running, it will launch one with the correct language.
		:type corenlp_url: str
		:type useConstituencyParserOnly: bool
		:type language: str
		"""

		acceptedLanguages = ['arabic','chinese','english','french','german','spanish']
		assert language in acceptedLanguages

		self.corenlp_url = corenlp_url

		self.language = language

		self.nlp = spacy.load('en', disable=['ner'])

	def _sentencesGenerator(self,text):
		parsed = self.nlp(text)
		sentence = None
		for token in parsed:
			if sentence is None or token.is_sent_start:
				if not sentence is None:
					yield sentence
				sentence = []
			sentence.append(token)

		if not sentence is None and len(sentence) > 0:
			yield sentence

	def sentenceSplit(docTokens):
		result = []
		sentence = []
		#for token in parsed:
		for token in docTokens:
			if token.is_sent_start:
				result.append(sentence)
				sentence = []
			sentence.append(token)

		if len(sentence) > 0:
			result.append(sentence)

		return sentenceSplit

	def parse(self,corpus):
		"""
		Parse the corpus. Each document will be split into sentences which are then tokenized and parsed for their dependency graph. All parsed information is stored within the corpus object.
		
		:param corpus: Corpus to parse
		:type corpus: kindred.Corpus
		"""

		assert isinstance(corpus,kindred.Corpus)

		for d in corpus.documents:
		#for doctokens in self.nlp.pipe( d.text for d in corpus.documents, batch_size=2, n_threads=1):
			entityIDsToEntities = d.getEntityIDsToEntities()
		
			denotationTree = IntervalTree()
			entityTypeLookup = {}
			for e in d.getEntities():
				entityTypeLookup[e.entityID] = e.entityType
			
				for a,b in e.position:
					denotationTree[a:b] = e.entityID
				
			for sentence in self._sentencesGenerator(d.text):
				tokens = []
				for t in sentence:
					token = kindred.Token(t.text,t.lemma_,t.pos_,t.idx,t.idx+len(t.text))
					tokens.append(token)

				sentenceStart = tokens[0].startPos
				sentenceEnd = tokens[-1].endPos
				sentenceTxt = d.text[sentenceStart:sentenceEnd]

				indexOffset = sentence[0].i
				dependencies = []
				for t in sentence:
					depName = t.dep_
					dep = (t.head.i-indexOffset,t.i-indexOffset,depName)
					dependencies.append(dep)

				# TODO: Should I filter this more or just leave it for simplicity
					
				entityIDsToTokenLocs = defaultdict(list)
				for i,t in enumerate(tokens):
					entitiesOverlappingWithToken = denotationTree[t.startPos:t.endPos]
					for interval in entitiesOverlappingWithToken:
						entityID = interval.data
						entityIDsToTokenLocs[entityID].append(i)

				# Let's gather up the information about the "known" entities in the sentence
				entitiesWithLocations = []
				for entityID,entityLocs in sorted(entityIDsToTokenLocs.items()):
					e = entityIDsToEntities[entityID]
					entityWithLocation = (e, entityLocs)
					entitiesWithLocations.append(entityWithLocation)
					
				sentence = kindred.Sentence(sentenceTxt, tokens, dependencies, entitiesWithLocations, d.getSourceFilename())
				d.addSentence(sentence)

		corpus.parsed = True


@@ -233,57 +233,8 @@
 elf,
-corenlp_url=None,useConstituencyParserOnly=False,
 lang
@@ -241,21 +241,16 @@
 uage='en
-glish
 '):%0A%09%09%22%22
@@ -290,23 +290,21 @@
 ill use 
-CoreNLP
+Spacy
  for par
@@ -311,251 +311,47 @@
 sing
-%0A%09%09%0A%09%09:param corenlp_url: URL of the CoreNLP instance%0A%09%09:param useConstituencyParserOnly: Use CoreNLP's constituency parser (with a conversion) for the depenency parse information, and not the CoreNLP dependency parser. This is slower%0A%09%09:par
+. It uses Spacy and offers all the s
 am
+e
  lan
@@ -359,483 +359,447 @@
 uage
-: Language to parse (english/arabic/chinese/french/german/spanish). The parser will check that the current CoreNLP is using the matching language. If no CoreNLP instance is running, it will launch one with the correct language.%0A%09%09:type corenlp_url: str%0A%09%09:type useConstituencyParserOnly: bool%0A%09%09:type language: str%0A%09%09%22%22%22%0A%0A%09%09acceptedLanguages = %5B'arabic','chinese','english','french','german','spanish'%5D%0A%09%09assert language in acceptedLanguages%0A%0A%09%09self.corenlp_url = corenlp_url
+s that Spacy offers. Check out: https://spacy.io/usage/models. Note that the language model needs to be downloaded first (e.g. python -m spacy download en)%0A%09%09%0A%09%09:param language: Language to parse (en/de/es/pt/fr/it/nl)%0A%09%09:type language: str%0A%09%09%22%22%22%0A%0A%09%09acceptedLanguages = %5B'en','de','es','pt','fr','it','nl'%5D%0A%09%09assert language in acceptedLanguages, %22Language for parser (%25s) not in accepted languages: %25s%22 %25 (language,str(acceptedLanguages))
 %0A%0A%09%09
@@ -848,20 +848,24 @@
 cy.load(
-'en'
+language
 , disabl
@@ -1194,16 +1194,17 @@
 e%0A%0A%09def 
+_
 sentence

Add extra languages to Parser and tidy