from __future__ import division
from percept.tasks.base import Task
from percept.tasks.train import Train
from percept.fields.base import Complex, List, Dict, Float
from inputs.inputs import SimpsonsFormats
from percept.utils.models import RegistryCategories, get_namespace
import logging
import numpy as np
import calendar
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
import math
import random
from itertools import chain
from percept.tests.framework import Tester
import os
from percept.conf.base import settings

log = logging.getLogger(__name__)

def make_df(datalist, labels, name_prefix=""):
    df = pd.DataFrame(datalist).T
    if name_prefix!="":
        labels = [name_prefix + "_" + l for l in labels]
    labels = [l.replace(" ", "_").lower() for l in labels]
    df.columns = labels
    df.index = range(df.shape[0])
    return df

class CleanupScriptList(Task):
    data = Complex()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data)

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """

        script_removal_values = [""]
        for r in script_removal_values:
            data = data[data["script"]!=r]

        return data

class CleanupScriptText(Task):
    data = Complex()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data)

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """

        for i in xrange(0,data.shape[0]):
            pass

        return data
@@ -533,16 +533,26 @@
 settings
+%0Aimport re
 %0A%0Alog = 
@@ -580,16 +580,49 @@
 ame__)%0A%0A
+PUNCTUATION = %5B%22%5D%22,%22.%22,%22!%22,%22?%22%5D%0A%0A
 def make
@@ -1542,16 +1542,258 @@
 t%22%5D!=r%5D%0A
+        log.info(data)%0A        data%5B'episode_name'%5D = %5Bi.split('%5Cn')%5B0%5D.strip() for i in data%5B'episode_name'%5D%5D%0A        data%5B'episode_code'%5D = %5Bi.split('/')%5B-1%5D.split('.html')%5B0%5D for i in data%5B'url'%5D%5D%0A%0A        data.index = range(data.shape%5B0%5D)
 %0A       
@@ -2313,32 +2313,59 @@
 de%0A        %22%22%22%0A%0A
+        voice_scripts = %5B%5D%0A
         for i in
@@ -2406,11 +2406,873 @@
     
-pas
+script_lines = data%5B'script'%5D%5Bi%5D.split('%5Cn')%0A            voice_lines = %5B%5D%0A            current_line = %22%22%0A            for (i,line) in enumerate(script_lines):%0A                line = line.strip()%0A                if line.startswith(%22%5B%22) and line.endswith(%22%5D%22):%0A                    continue%0A                voice_line = re.search('%5Cw+:',line)%0A                if voice_line is not None:%0A                    voice_lines.append(current_line)%0A                    current_line = line%0A                elif len(line)==0 and len(current_line)%3E0:%0A                    voice_lines.append(current_line)%0A                    current_line = %22%22%0A                elif len(current_line)%3E0:%0A                    current_line+=line%0A            voice_scripts.append(%22%5Cn%22.join(%5Bl for l in voice_lines if len(l)%3E0 and %22%7B%22 not in l and %22=%22 not in l%5D))%0A%0A        data%5B'voice_script'%5D = voice_script
 s%0A%0A 

Extract only the voice lines