""" Tokenization utilities
"""

__author__ = ['Patrick J. Burns <patrick@diyclassics.org>']
__license__ = 'MIT License.'

import pickle
from abc import abstractmethod

from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktTrainer
from nltk.tokenize.punkt import PunktLanguageVars

class BaseSentenceTokenizerTrainer(object):
    """ Train sentence tokenizer
    """

    def __init__(self, language=None, punctuation=None, strict=None, abbreviations=None):
        """ Initialize stoplist builder with option for language specific parameters
        :type language: str
        :param language : text from which to build the stoplist
        """
        if language:
            self.language = language.lower()

        self.punctuation = punctuation
        self.strict = strict
        self.abbreviations = abbreviations

    def train_sentence_tokenizer(self, text):
        """
        Train sentence tokenizer.
        """

        # Set punctuation
        language_punkt_vars = PunktLanguageVars

        if self.punctuation or self.strict:
            language_punkt_vars.sent_end_chars = self.punctuation+self.strict

        # Set abbreviations
        trainer = PunktTrainer(text, language_punkt_vars)
        trainer.INCLUDE_ALL_COLLOCS = True
        trainer.INCLUDE_ABBREV_COLLOCS = True

        tokenizer = PunktSentenceTokenizer(trainer.get_params())

        for abbreviation in self.abbreviations:
            tokenizer._params.abbrev_types.add(abbreviation)

        return tokenizer

    def pickle_sentence_tokenizer(self, filename, tokenizer):
        # Dump pickled tokenizer
        with open(filename, 'wb') as f:
            pickle.dump(tokenizer, f)

@@ -929,24 +929,78 @@
     %22%22%22%0A
-%0A
        
+ print(self.punctuation)%0A        return None%0A%0A        #
  # Set p
@@ -1013,24 +1013,26 @@
 tion%0A       
+ #
  language_pu
@@ -1060,24 +1060,33 @@
 ageVars%0A
+        #
 %0A       
  if self
@@ -1073,24 +1073,26 @@
    #%0A       
+ #
  if self.pun
@@ -1119,24 +1119,26 @@
 ict:%0A       
+ #
      languag
@@ -1196,16 +1196,25 @@
 .strict%0A
+        #
 %0A       
@@ -1215,16 +1215,18 @@
        #
+ #
  Set abb
@@ -1235,32 +1235,34 @@
 viations%0A       
+ #
  trainer = Punkt
@@ -1295,32 +1295,34 @@
 kt_vars)%0A       
+ #
  trainer.INCLUDE
@@ -1348,16 +1348,18 @@
 %0A       
+ #
  trainer
@@ -1389,24 +1389,33 @@
  = True%0A
+        #
 %0A       
  tokeniz
@@ -1402,24 +1402,26 @@
    #%0A       
+ #
  tokenizer =
@@ -1466,24 +1466,33 @@
 rams())%0A
+        #
 %0A       
  for abb
@@ -1483,16 +1483,18 @@
 %0A       
+ #
  for abb
@@ -1529,24 +1529,26 @@
 ons:%0A       
+ #
      tokeniz
@@ -1589,24 +1589,33 @@
 iation)%0A
+        #
 %0A       
  return 
@@ -1598,32 +1598,34 @@
        #%0A       
+ #
  return tokenize

Test inheritance