import abc
from nalaf.structures.data import Token
import re
from nltk.tokenize import word_tokenize


class Tokenizer:
    """
    Abstract class for splitting the raw text (or sentences if Splitter was called first)
    into tokens for each document in the dataset.
    Subclasses that inherit this class should:
    * Be named [Name]Tokenizer
    * Implement the abstract method tokenize
    * Append new sub-items to each list of the list field "sentences" of each Part in the dataset
    """

    @abc.abstractmethod
    def tokenize(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        return


class GenericTokenizer(Tokenizer):

    def __init__(self, string_splitter_fun):
        self.string_splitter_fun = string_splitter_fun
        "A function that takes a string as input and returns a list/iterator of tokenized string items"


    def tokenize(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        for part in dataset.parts():
            so_far = 0
            part.sentences = []
            for index, sentence_ in enumerate(part.sentences_):
                part.sentences.append([])

                for token_word in self.string_splitter_fun(sentence_):
                    token_start = part.text.find(token_word, so_far)
                    so_far = token_start + len(token_word)
                    part.sentences[index].append(Token(token_word, token_start))


NLTK_TOKENIZER = GenericTokenizer(word_tokenize)


class TmVarTokenizer(Tokenizer):
    """
    Implementation of the TmVar tokenizer as descriped in their paper. Code ported from perl.
    Requires
    """
    def tokenize(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        for part in dataset.parts():
            so_far = 0
            part.sentences = []
            for index, sentence_ in enumerate(part.sentences_):
                sentence = sentence_
                sentence = re.sub('([0-9])([A-Za-z])', r'\1 \2', sentence)
                # removing for now split from Capital to Lower
                # no noticeable difference in performance but we keep whole words
                # sentence = re.sub('([A-Z])([a-z])', r'\1 \2', sentence)
                sentence = re.sub('([a-z])([A-Z])', r'\1 \2', sentence)
                sentence = re.sub('([A-Za-z])([0-9])', r'\1 \2', sentence)
                sentence = re.sub('([a-z])(fs)', r'\1 \2', sentence)

                # separate non-ascii characters into their own tokens
                sentence = re.sub('([^\x00-\x7F])', r' \1 ', sentence)

                sentence = re.sub('([\W\-_])', r' \1 ', sentence)

                part.sentences.append([])
                for token_word in sentence.split():
                    token_start = part.text.find(token_word, so_far)
                    so_far = token_start + len(token_word)
                    part.sentences[index].append(Token(token_word, token_start))

@@ -491,16 +491,151 @@
    %22%22%22%0A%0A
+    @abc.abstractmethod%0A    def tokenize_string(self, string):%0A        %22%22%22%0A        :return string iterable%0A        %22%22%22%0A        return%0A%0A
     @abc
@@ -1019,16 +1019,105 @@
 tems%22%0A%0A%0A
+    def tokenize_string(self, string):%0A        return self.string_splitter_fun(string)%0A%0A%0A
     def 
@@ -1456,35 +1456,31 @@
 in self.
-string_splitter_fun
+tokenize_string
 (sentenc
@@ -1695,16 +1695,18 @@
 art))%0A%0A%0A
+%0A%0A
 NLTK_TOK
@@ -1900,24 +1900,25 @@
 res%0A    %22%22%22%0A
+%0A
     def toke
@@ -1925,266 +1925,31 @@
 nize
-(self, dataset):%0A        %22%22%22%0A        :type dataset: nalaf.structures.data.Dataset%0A        %22%22%22%0A        for part in dataset.parts():%0A            so_far = 0%0A            part.sentences = %5B%5D%0A            for index, sentence_ in enumerate(part.sentences_):%0A        
+_string(self, string):%0A
     
@@ -1968,28 +1968,22 @@
  = s
-entence_%0A   
+tring%0A
         
     
@@ -1970,37 +1970,32 @@
  string%0A        
-     
 sentence = re.su
@@ -2045,24 +2045,16 @@
         
-      
-  
 # removi
@@ -2092,24 +2092,16 @@
 o Lower%0A
-        
         
@@ -2170,32 +2170,24 @@
 rds%0A        
- 
-       
 # sentence =
@@ -2236,32 +2236,24 @@
 ce)%0A        
-    
-    
 sentence = r
@@ -2300,32 +2300,24 @@
 ce)%0A        
-      
-  
 sentence = r
@@ -2363,24 +2363,16 @@
 ntence)%0A
-        
         
@@ -2433,24 +2433,16 @@
         
- 
-       
 # separa
@@ -2491,32 +2491,24 @@
 ens%0A        
-    
-    
 sentence = r
@@ -2559,24 +2559,16 @@
         
-       
- 
 sentence
@@ -2602,32 +2602,357 @@
 1 ', sentence)%0A%0A
+        return sentence.split()  # splits by white space%0A%0A%0A    def tokenize(self, dataset):%0A        %22%22%22%0A        :type dataset: nalaf.structures.data.Dataset%0A        %22%22%22%0A        for part in dataset.parts():%0A            so_far = 0%0A            part.sentences = %5B%5D%0A            for index, sentence_ in enumerate(part.sentences_):%0A
                 
@@ -2977,16 +2977,17 @@
 end(%5B%5D)%0A
+%0A
         
@@ -3014,29 +3014,44 @@
 rd in se
-ntence.split(
+lf.tokenize_string(sentence_
 ):%0A     

generalize tokenizers to tokenize any string