#coding=utf8

from __future__ import unicode_literals
import re, codecs
from .Normalizer import *
from .WordTokenizer import *

default_pos_map = {'ADJ': 'ADJ', 'ADJ_CMPR': 'ADJ', 'ADJ_INO': 'ADJ', 'ADJ_ORD': 'ADJ', 'ADJ_SIM': 'ADJ', 'ADJ_SUP': 'ADJ', 'ADV': 'ADV', 'ADV_EXM': 'ADV', 'ADV_I': 'ADV', 'ADV_NEGG': 'ADV', 'ADV_NI': 'ADV', 'ADV_TIME': 'ADV', 'AR': 'AR', 'CON': 'CONJ', 'DEFAULT': 'DEFAULT', 'DELM': 'PUNC', 'DET': 'PREP', 'IF': 'IF', 'INT': 'INT', 'MORP': 'MORP', 'MQUA': 'MQUA', 'MS': 'MS', 'N_PL': 'N', 'N_SING': 'N', 'NN': 'NN', 'NP': 'NP', 'OH': 'OH', 'OHH': 'OHH', 'P': 'PREP', 'PP': 'PP', 'PRO': 'PR', 'PS': 'PS', 'QUA': 'QUA', 'SPEC': 'SPEC', 'V_AUX': 'V', 'V_IMP': 'V', 'V_PA': 'V', 'V_PRE': 'V', 'V_PRS': 'V', 'V_SUB': 'V'}


class BijankhanReader():
	def __init__(self, bijankhan_file='resources/bijankhan.txt', joined_verb_parts=True, pos_map=default_pos_map):
		"""
			downnload bijankhan corpus from ...
		"""
		self._bijankhan_file = bijankhan_file
		self._joined_verb_parts = joined_verb_parts
		self._pos_map = pos_map
		self._normalizer = Normalizer(punctuation_spacing=False)
		self._tokenizer = WordTokenizer()
		self._map_items = lambda item: (self._normalizer.normalize(item[0]), self._pos_map.get(item[1], item[1]))

	def _sentences(self):
		sentence = []
		for line in codecs.open(self._bijankhan_file, encoding='utf-8'):
			parts = re.split('  +', line.strip())
			if len(parts) == 2:
				word, tag = parts
				if word != '#':
					sentence.append((word, tag))
				if tag == 'DELM':
					if len(sentence):
						yield sentence
						sentence = []

	def _join_verb_parts(self, sentence):
		"""
		>>> bijankhan._join_verb_parts([('اولين', 'ADJ_SUP'), ('سياره', 'N_SING'), ('خارج', 'ADJ_SIM'), ('از', 'P'), ('منظومه', 'N_SING'), ('شمسي', 'ADJ_SIM'), ('ديده', 'ADJ_INO'), ('شد', 'V_PA'), ('.', 'DELM')])
		[('اولين', 'ADJ_SUP'), ('سياره', 'N_SING'), ('خارج', 'ADJ_SIM'), ('از', 'P'), ('منظومه', 'N_SING'), ('شمسي', 'ADJ_SIM'), ('ديده شد', 'V_PA'), ('.', 'DELM')]
		"""
		return sentence

	def sents(self):
		for sentence in self._sentences():
			if self._joined_verb_parts:
				sentence = self._join_verb_parts(sentence)
			yield list(map(self._map_items, sentence))


if __name__ == '__main__':
	import doctest
	doctest.testmod(extraglobs={'bijankhan': BijankhanReader()})

@@ -1148,20 +1148,20 @@
 lf._map_
-item
+pose
 s = lamb
@@ -1173,43 +1173,16 @@
 em: 
-(self._normalizer.normalize
 (item%5B0%5D
 ), s
@@ -1177,17 +1177,16 @@
 (item%5B0%5D
-)
 , self._
@@ -1451,20 +1451,48 @@
 append((
+self._normalizer.normalize(
 word
+)
 , tag))%0A
@@ -1658,25 +1658,25 @@
 parts(%5B('%D8%A7%D9%88%D9%84
-%D9%8A
+%DB%8C
 %D9%86', 'ADJ_SUP
@@ -1674,33 +1674,33 @@
  'ADJ_SUP'), ('%D8%B3
-%D9%8A
+%DB%8C
 %D8%A7%D8%B1%D9%87', 'N_SING'),
@@ -1753,33 +1753,33 @@
 'N_SING'), ('%D8%B4%D9%85%D8%B3
-%D9%8A
+%DB%8C
 ', 'ADJ_SIM'), (
@@ -1780,17 +1780,17 @@
 M'), ('%D8%AF
-%D9%8A
+%DB%8C
 %D8%AF%D9%87', 'AD
@@ -1838,17 +1838,17 @@
 %09%09%5B('%D8%A7%D9%88%D9%84
-%D9%8A
+%DB%8C
 %D9%86', 'ADJ
@@ -1858,17 +1858,17 @@
 P'), ('%D8%B3
-%D9%8A
+%DB%8C
 %D8%A7%D8%B1%D9%87', 'N
@@ -1937,17 +1937,17 @@
 ), ('%D8%B4%D9%85%D8%B3
-%D9%8A
+%DB%8C
 ', 'ADJ_
@@ -1960,9 +1960,9 @@
  ('%D8%AF
-%D9%8A
+%DB%8C
 %D8%AF%D9%87 %D8%B4
@@ -2175,20 +2175,20 @@
 lf._map_
-item
+pose
 s, sente

fix bijankhan reader unit test