"""
Phantom phout format reader. Read chunks from phout and produce data frames
"""
import pandas as pd
from StringIO import StringIO
import logging
import json
import time
import datetime

logger = logging.getLogger(__name__)

phout_columns = [
    'send_ts', 'tag', 'interval_real', 'connect_time', 'send_time', 'latency',
    'receive_time', 'interval_event', 'size_out', 'size_in', 'net_code',
    'proto_code'
]


def string_to_df(data):
    start_time = time.time()
    chunk = pd.read_csv(
        StringIO(data),
        sep='\t',
        names=phout_columns,
        engine='python')
    chunk['receive_ts'] = chunk.send_ts + chunk.interval_real / 1e6
    chunk['receive_sec'] = chunk.receive_ts.astype(int)
    # TODO: consider configuration for the following:
    chunk['tag'] = chunk.tag.str.rsplit('#', 1, expand=True)[0]
    chunk.set_index(['receive_sec'], inplace=True)

    logger.info("Chunk decode time: %.2fms", (time.time() - start_time) * 1000)
    return chunk


class PhantomReader(object):
    def __init__(self, filename):
        self.buffer = ""
        self.stat_buffer = ""
        self.phout = open(filename, 'r')
        self.closed = False

    def next(self):
        if self.closed:
            raise StopIteration
        data = self.phout.read(1024 * 1024 * 10)
        if data:
            parts = data.rsplit('\n', 1)
            if len(parts) > 1:
                ready_chunk = self.buffer + parts[0] + '\n'
                self.buffer = parts[1]
                return string_to_df(ready_chunk)
            else:
                self.buffer += parts[0]
        else:
            self.phout.readline()
        return None

    def __iter__(self):
        return self

    def close(self):
        self.closed = True
        self.phout.close()


class PhantomStatsReader(object):
    def __init__(self, filename, phantom_info):
        self.phantom_info = phantom_info
        self.buffer = ""
        self.stat_buffer = ""
        self.stat_filename = filename
        self.closed = False
        self.start_time = 0

    def __decode_stat_data(self, chunk):
        for date_str, statistics in chunk.iteritems():
            date_obj = datetime.datetime.strptime(
                date_str.split(".")[0], '%Y-%m-%d %H:%M:%S')
            chunk_date = int(time.mktime(date_obj.timetuple()))
            instances = 0
            for benchmark_name, benchmark in statistics.iteritems():
                if not benchmark_name.startswith("benchmark_io"):
                    continue
                for method, meth_obj in benchmark.iteritems():
                    if "mmtasks" in meth_obj:
                        instances += meth_obj["mmtasks"][2]

            offset = chunk_date - 1 - self.start_time
            reqps = 0
            if offset >= 0 and offset < len(self.phantom_info.steps):
                reqps = self.phantom_info.steps[offset][0]
            yield {'ts': chunk_date - 1,
                   'metrics': {'instances': instances,
                               'reqps': reqps}}

    def __iter__(self):
        """
        Union buffer and chunk, split using '\n},',
        return splitted parts
        """
        self.start_time = int(time.time())
        logger.debug("Phantom steps:\n%s", self.phantom_info.steps)
        with open(self.stat_filename, 'r') as stat_file:
            while not self.closed:
                chunk = stat_file.read(1024 * 1024 * 10)
                if chunk:
                    self.stat_buffer += chunk
                    parts = self.stat_buffer.rsplit('\n},', 1)
                    if len(parts) > 1:
                        ready_chunk = parts[0]
                        self.stat_buffer = parts[1]
                        for m in ready_chunk.split('\n},'):
                            yield list(self.__decode_stat_data(json.loads(
                                '{%s}}' % m)))
                else:
                    stat_file.readline()
                yield None

    def close(self):
        self.closed = True

@@ -97,16 +97,35 @@
 s as pd%0A
+import numpy as np%0A
 from Str
@@ -430,16 +430,353 @@
 ode'%0A%5D%0A%0A
+dtypes = %7B%0A    'time': np.float64,%0A    'tag': np.str,%0A    'interval_real': np.int64,%0A    'connect_time': np.int64,%0A    'send_time': np.int64,%0A    'latency': np.int64,%0A    'receive_time': np.int64,%0A    'interval_event': np.int64,%0A    'size_out': np.int64,%0A    'size_in': np.int64,%0A    'net_code': np.int64,%0A    'proto_code': np.int64,%0A%7D%0A%0A
 %0Adef str
@@ -929,24 +929,22 @@
     
-engine='python')
+dtype=dtypes)%0A
 %0A   

use dtypes