#!/usr/bin/env python
# -*- coding: utf-8 -*-


SENT_MARK = "</s>"
UNK_MARK = "<unk>"

import numpy as np
import logging as loggers

logging = loggers.getLogger(__name__)


class Vocab(object):

    def __init__(self, is_lang=True, char_based=False):
        self.vocab_map = {}
        self.reversed_map = None
        self.size = 0
        self._char_based = char_based
        if is_lang:
            self.add(SENT_MARK)
            self.add(UNK_MARK)

    def add(self, word):
        if word not in self.vocab_map:
            self.vocab_map[word] = self.size
            self.size += 1

    def index(self, word):
        if word in self.vocab_map:
            return self.vocab_map[word]
        else:
            return self.vocab_map[UNK_MARK]

    def word(self, index):
        if not self.reversed_map:
            self.reversed_map = {}
            for k in self.vocab_map:
                self.reversed_map[self.vocab_map[k]] = k
        return self.reversed_map[index]

    def transform(self, word):
        v = np.zeros(self.size, dtype=int)
        v[self.index(word)] = 1
        return v

    def transform_index(self, index):
        v = np.zeros(self.size, dtype=int)
        v[index] = 1
        return v

    def _load_fixed_size(self, path, max_size):
        from collections import Counter
        logging.info("fixed size: %d" % max_size)
        counter = Counter()
        for line in open(path).readlines():
            line = line.strip()
            words = line.split(" ") if not self._char_based else line
            counter.update(words)
        for w, _ in counter.most_common(max_size):
            self.add(w)

    def load(self, path, max_size=-1):
        logging.info("load data from %s" % path)
        if max_size > 0:
            self._load_fixed_size(path, max_size)
            return
        for line in open(path).xreadlines():
            line = line.strip()
            words = line.split(" ") if not self._char_based else line
            map(self.add, words)
        logging.info("vocab size: %d" % self.size)

    @property
    def sent_index(self):
        return 0

    @property
    def sent_vector(self):
        return self.transform(SENT_MARK)



@@ -60,16 +60,37 @@
  %22%3C/s%3E%22%0A
+NULL_MARK = %22%3Cnull%3E%22%0A
 UNK_MARK
@@ -406,16 +406,48 @@
 s_lang:%0A
+            self.add(NULL_MARK)%0A
         
@@ -2166,17 +2166,37 @@
  return 
-0
+self.index(SENT_MARK)
 %0A%0A    @p

fix vocab