import csv
import pandas as pd


WORD_COL_NAME = "word"
TYPE_COL_NAME = "type"
IDENTITY_TYPE = "identity"
CONNOTATION_COL_NAME = "connotation"
NEUTRAL_CONNOTATION = "neutral"
TOXIC_CONNOTATION = "toxic"


class Madlibber(object):

  def __init__(self, path_helper, format_helper, word_helper):
    self.path_helper = path_helper
    self.format_helper = format_helper
    self.word_helper = word_helper
    self.__templates = []
    self.__template_elements = set([])
    self.__template_word_categories = set([])
    self.__toxicity = set(["toxic", "nontoxic"])

  def check_duplicates_sentence_templates(self):
    df = pd.read_csv(self.path_helper.sentence_template_file)
    if df[df.duplicated()]:
      raise ValueError("Duplicate sentence templates")

  def load_sanity_check_templates_and_infer_word_categories(self):
    print("Loading templates, sanity checking and inferring word categories...")
    self.check_duplicates_sentence_templates()
    f = open(self.path_helper.sentence_template_file, "r")
    csv_f = csv.DictReader(f)

    for line in csv_f:
      template = line["template"]
      toxicity = line["toxicity"]
      phrase = line["phrase"]
      template_elements = self.format_helper.extract_template_elements(phrase)

      if not phrase:
        continue

      if toxicity not in self.__toxicity:
        raise ValueError("'{}' is not a valid toxicity".format(toxicity))

      if not template_elements:
        raise ValueError("Template '{}' does not require fill-in: '{}'".format(
            template, phrase))

      for t in template_elements:
        template_element_word_categories = (
            self.format_helper.decompose_template_element(t))
        self.__template_word_categories.update(template_element_word_categories)

      self.__templates.append((template, toxicity, phrase, template_elements))
      self.__template_elements.update(template_elements)
    f.close()

    print("Inferred word categories: {}".format(", ".join(
        self.__template_word_categories)))
    print("Done")

  def load_and_sanity_check_words(self):
    print("Loading word list...")
    f = open(self.path_helper.word_file, "r")
    csv_f = csv.reader(f)
    header = next(csv_f)[:-1]  # Last column is the word itself
    words = []
    for line in csv_f:
      word = line[-1]  # Last column is the word itself
      if not word:
        continue
      words.append(word)
      for i, l in enumerate(line[:-1]):
        if l:
          word_category = self.format_helper.construct_word_category(
              header[i], l)
          self.word_helper.add_word(word_category, word)
    f.close()
    if len(set(words)) != len(words):
      raise ValueError("Duplicate words are not allowed.")

    for t in self.__template_word_categories:
      if t not in self.word_helper.word_categories:
        raise ValueError("Template word_category '{}' is not represented "
                         "in the word list".format(t))

    print("Word categories: {}".format(", ".join(
        self.word_helper.word_categories)))
    print("Done")

  def display_statistics(self):
    print("Template statistics:")
    print("Total number of templates: {}".format(len(self.__templates)))
    print("Total number of unique template elements to fill in: {}".format(
        len(self.__template_elements)))
    print("Word statistics:")
    template_element_word_counts = {}
    for i, te in enumerate(self.__template_elements):
      words = self.word_helper.get_template_element_words(te)
      template_element_word_counts[te] = len(words)
      print("Template element {}: {}, Total number of words: {}".format(
          i, te, template_element_word_counts[te]))
    total = 0
    for t in self.__templates:
      template_elements = t[-1]
      template_total = 1
      for te in template_elements:
        template_total *= template_element_word_counts[te]
      total += template_total
    print("Number of expected output lines: {}".format(total))

  def fill_templates(self):
    fout = open(self.path_helper.output_file, "w")
    csv_fout = csv.writer(fout)
    csv_fout.writerow(["template", "toxicity", "phrase"])

    count = 0
    for template, toxicity, phrase, template_elements in self.__templates:
      template_count = 0
      print(
          "Working on template '{}' with toxicity '{}' and phrase '{}'".format(
              template, toxicity, phrase))

      unicode_phrase = phrase.decode("utf-8")
      for words in self.__iterate_words(template_elements, 0):
        output_phrase = unicode_phrase.format(**words)
        csv_fout.writerow([template, toxicity, output_phrase.encode("utf-8")])
        count += 1
        template_count += 1

      print("Output {} sentences for template '{}', toxicity '{}'".format(
          template_count, template, toxicity))
    fout.close()

    print("Output {} total sentences".format(count))

  def __iterate_words(self, template_elements, template_element_index):
    is_last = template_element_index == (len(template_elements) - 1)
    template_element = template_elements[template_element_index]
    for word in self.word_helper.get_template_element_words(template_element):
      if is_last:
        yield {template_element: word}
      else:
        for words in self.__iterate_words(template_elements,
                                          template_element_index + 1):
          words[template_element] = word
          yield words

@@ -675,16 +675,20 @@
 %0A    if 
+not 
 df%5Bdf.du
@@ -698,16 +698,22 @@
 cated()%5D
+.empty
 :%0A      
@@ -4403,54 +4403,8 @@
 ))%0A%0A
-      unicode_phrase = phrase.decode(%22utf-8%22)%0A
     
@@ -4490,16 +4490,8 @@
 e = 
-unicode_
 phra
@@ -4573,24 +4573,8 @@
 rase
-.encode(%22utf-8%22)
 %5D)%0A 

Update madlibber.py