

import re
import textplot.utils as utils
import requests
import numpy as np

from nltk.stem import PorterStemmer
from collections import OrderedDict
from random import shuffle, randint
from itertools import islice


class Text(object):


    @classmethod
    def from_file(cls, path):

        """
        Create a text from a filepath.

        :param cls: The Text class.
        :param path: The filepath.
        """

        return cls(open(path, 'r').read())


    @classmethod
    def from_url(cls, url):

        """
        Create a text from a URL.

        :param cls: The Text class.
        :param path: The URL.
        """

        return cls(requests.get(url).text)


    def __init__(self, text):

        """
        Store and tokenize the text.

        :param text: The text as a raw string.
        """

        self.text = text
        self.stem = PorterStemmer().stem
        self.tokenize()


    def tokenize(self):

        """
        Tokenize the text.
        """

        self.tokens = []
        self.terms = OrderedDict()

        # Strip tags and downcase.
        text = utils.strip_tags(self.text).lower()

        # Walk words in the text.
        pattern = re.compile('[a-z]+')
        for i, match in enumerate(re.finditer(pattern, text)):

            # Stem the token.
            token = match.group(0)
            stemmed = self.stem(token)

            # Token:
            self.tokens.append({
                'token':    token,
                'stemmed':  stemmed,
                'left':     match.start(),
                'right':    match.end()
            })

            # Token -> offset:
            if stemmed in self.terms: self.terms[stemmed].append(i)
            else: self.terms[stemmed] = [i]


    def get_count_difference(self, word1, word2):

        """
        What is the difference between the number of instances of two words?

        :param word1: The first word.
        :param word2: The other word.
        """

        return abs(len(self.terms[word1]) - len(self.terms[word2]))


    def get_similarity(self, anchor, sample):

        """
        Given an "anchor" word, compute a similarity metric for another word.

        :param anchor: The word to compare against.
        :param sample: The word being compared.
        """

        distances = []

        anchor_offsets = np.array(self.terms[anchor])
        sample_offsets = np.array(self.terms[sample])

        for offset in anchor_offsets:

            # Get the closest instance of the sample.
            cidx = np.abs(sample_offsets - offset).argmin()

            # Get the distance between the two.
            distance = abs(sample_offsets[cidx] - offset)
            distances.append(distance)

        return np.mean(np.array(distances))


    def get_similarities(self, anchor):

        """
        Given an "anchor" word, compute a sorted list of similarities for all
        other terms in the text.

        :param anchor: The word to compare against.
        """

        sims = []
        anchor = self.stem(anchor)

        for term in self.terms:
            if self.get_count_difference(anchor, term) < 500:
                sims.append((term, self.get_similarity(term, anchor)))

        sims.sort(key=lambda s: s[1])
        return sims

@@ -1316,21 +1316,24 @@
         
-token
+original
  = match
@@ -1374,21 +1374,24 @@
 lf.stem(
-token
+original
 )%0A%0A     
@@ -1460,24 +1460,27 @@
    '
-token':    token
+original': original
 ,%0A  
@@ -1712,16 +1712,16 @@
 pend(i)%0A
+
         
@@ -1760,1544 +1760,4 @@
 %5Bi%5D%0A
-%0A%0A    def get_count_difference(self, word1, word2):%0A%0A        %22%22%22%0A        What is the difference between the number of instances of two words?%0A%0A        :param word1: The first word.%0A        :param word2: The other word.%0A        %22%22%22%0A%0A        return abs(len(self.terms%5Bword1%5D) - len(self.terms%5Bword2%5D))%0A%0A%0A    def get_similarity(self, anchor, sample):%0A%0A        %22%22%22%0A        Given an %22anchor%22 word, compute a similarity metric for another word.%0A%0A        :param anchor: The word to compare against.%0A        :param sample: The word being compared.%0A        %22%22%22%0A%0A        distances = %5B%5D%0A%0A        anchor_offsets = np.array(self.terms%5Banchor%5D)%0A        sample_offsets = np.array(self.terms%5Bsample%5D)%0A%0A        for offset in anchor_offsets:%0A%0A            # Get the closest instance of the sample.%0A            cidx = np.abs(sample_offsets - offset).argmin()%0A%0A            # Get the distance between the two.%0A            distance = abs(sample_offsets%5Bcidx%5D - offset)%0A            distances.append(distance)%0A%0A        return np.mean(np.array(distances))%0A%0A%0A    def get_similarities(self, anchor):%0A%0A        %22%22%22%0A        Given an %22anchor%22 word, compute a sorted list of similarities for all%0A        other terms in the text.%0A%0A        :param anchor: The word to compare against.%0A        %22%22%22%0A%0A        sims = %5B%5D%0A        anchor = self.stem(anchor)%0A%0A        for term in self.terms:%0A            if self.get_count_difference(anchor, term) %3C 500:%0A                sims.append((term, self.get_similarity(term, anchor)))%0A%0A        sims.sort(key=lambda s: s%5B1%5D)%0A        return sims%0A

Store the original token.