
#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: dev_nb/Sentence_Weight.ipynb
import requests
from lexrank import LexRank
from lexrank.mappings.stopwords import STOPWORDS
import spacy
from spacy.tokens import Doc, Span
import numpy as np

urls= ['http://www.gutenberg.org/cache/epub/29444/pg29444.txt',
       'http://www.gutenberg.org/cache/epub/31428/pg31428.txt',
       'http://www.gutenberg.org/cache/epub/4908/pg4908.txt',
       'http://www.gutenberg.org/cache/epub/33504/pg33504.txt',
       'http://www.gutenberg.org/cache/epub/39017/pg39017.txt',
       'http://www.gutenberg.org/cache/epub/37157/pg37157.txt',
       'http://www.gutenberg.org/cache/epub/15207/pg15207.txt',
       'http://www.gutenberg.org/cache/epub/10773/pg10773.txt',
       'http://www.gutenberg.org/cache/epub/31624/pg31624.txt',
       'http://www.gutenberg.org/cache/epub/5192/pg5192.txt',
       'http://www.gutenberg.org/cache/epub/40030/pg40030.txt',
       'http://www.gutenberg.org/cache/epub/50992/pg50992.txt',
       'http://www.gutenberg.org/cache/epub/50880/pg50880.txt',
       'http://www.gutenberg.org/cache/epub/36525/pg36525.txt',
       'http://www.gutenberg.org/cache/epub/47167/pg47167.txt',
       'http://www.gutenberg.org/cache/epub/47464/pg47464.txt',
       'http://www.gutenberg.org/cache/epub/33397/pg33397.txt',
       'http://www.gutenberg.org/cache/epub/29782/pg29782.txt',
       'http://www.gutenberg.org/cache/epub/32857/pg32857.txt',
       'http://www.gutenberg.org/cache/epub/26262/pg26262.txt']
docs = [requests.get(u).text for u in urls]

lxr = LexRank(docs, stopwords=STOPWORDS['en'])
nlp =  spacy.load('en_core_web_md')


def tfidf(span):return sum([tok.prob for tok in span])*1.0/len(span)
Span.set_extension("weight", getter=tfidf, force=True)

def ranktfidf(doc): return np.argsort(np.array([sent._.weight for sent in doc.sents]))[::-1]
Doc.set_extension("ranktfidf", getter=ranktfidf, force=True)