from bs4 import BeautifulSoup
import urllib
import config as c
from pymongo import MongoClient

client = MongoClient()
db = client.articles.entries

def generate_filename(entry, directory = c.ARTICLE_DIR):
  authors = [a.split()[-1] for a in entry['authors']]
  authors = authors[0]+'_et.al' if len(authors) > 1 else authors[0]
  title = entry['title'].replace(' ', '_')
  return ''.join([directory, authors,'-',title , '.pdf'])

def dl_pdf_from_arxiv(url):
  html_doc = urllib.urlopen(url).read()
  s = BeautifulSoup(html_doc)
  entries = [{
    'pdf' : e.findAll('link',attrs={'type': 'application/pdf'})[0]['href'],
    'url' : e.findAll('link',attrs={'type': 'text/html'})[0]['href'],
    'authors': [str(a.next.next.next) for a in e.findAll('author')],
    'title': str(e.title.next),
    'id': str.split(str(e.id.next),'/')[-1],
            } for e in s.findAll('entry')]
  entries = filter(lambda e: db.find_one({'id': e['id']}) == None, entries)
  for entry in entries:
    entry['path'] = generate_filename(entry)
  map(lambda e: urllib.urlretrieve(e['pdf'], e['path']), entries)
  if entries:
    db.insert(entries)



@@ -142,16 +142,117 @@
 ntries%0A%0A
+def gen_index(seed=db.count()):%0A  i = seed%0A  while True:%0A    i +=1%0A    yield i%0A%0Aindex = gen_index()%0A%0A
 def gene
@@ -929,16 +929,41 @@
 ')%5B-1%5D,%0A
+    'index': next(index)%0A
         
@@ -1245,10 +1245,8 @@
 ntries)%0A
-%0A%0A

introduce index for entries