import cPickle as pickle
import urllib2
import shutil
import time
import os
import random

os.system('mkdir -p pdf') # ?

timeout_secs = 10 # after this many seconds we give up on a paper
numok = 0
numtot = 0
db = pickle.load(open('db.p', 'rb'))
have = set(os.listdir('pdf')) # get list of all pdfs we already have
for pid,j in db.iteritems():
  
  pdfs = [x['href'] for x in j['links'] if x['type'] == 'application/pdf']
  assert len(pdfs) == 1
  pdf_url = pdfs[0] + '.pdf'
  basename = pdf_url.split('/')[-1]
  fname = os.path.join('pdf', basename)

  # try retrieve the pdf
  numtot += 1
  try:
    if not basename in have:
      print 'fetching %s into %s' % (pdf_url, fname)
      req = urllib2.urlopen(pdf_url, None, timeout_secs)
      with open(fname, 'wb') as fp:
          shutil.copyfileobj(req, fp)
      time.sleep(0.1 + random.uniform(0,0.2))
    else:
      print '%s exists, skipping' % (fname, )
    numok+=1
  except Exception, e:
    print 'error downloading: ', pdf_url
    print e
  
  print '%d/%d of %d downloaded ok.' % (numok, numtot, len(db))
  
print 'final number of papers downloaded okay: %d/%d' % (numok, len(db))

@@ -4,26 +4,36 @@
 ort 
-cP
+p
 ickle
- as pickle%0A
+%0Afrom  urllib.request 
 impo
@@ -38,20 +38,20 @@
 port url
-lib2
+open
 %0Aimport 
@@ -98,39 +98,8 @@
 om%0A%0A
-os.system('mkdir -p pdf') # ?%0A%0A
 time
@@ -160,16 +160,148 @@
 a paper%0A
+%0Apdf_dir = os.path.join('data', 'pdf')%0Adb_file = os.path.join('data', 'db.p')%0Aif not os.path.exists(pdf_dir): os.makedirs(pdf_dir)%0A%0A
 numok = 
@@ -335,22 +335,23 @@
 ad(open(
-'
 db
-.p'
+_file
 , 'rb'))
@@ -373,21 +373,23 @@
 listdir(
-'
 pdf
-'
+_dir
 )) # get
@@ -442,12 +442,8 @@
  db.
-iter
 item
@@ -637,21 +637,23 @@
 th.join(
-'
 pdf
-'
+_dir
 , basena
@@ -743,17 +743,17 @@
    print
- 
+(
 'fetchin
@@ -784,16 +784,17 @@
 , fname)
+)
 %0A      r
@@ -802,16 +802,8 @@
 q = 
-urllib2.
 urlo
@@ -976,17 +976,17 @@
    print
- 
+(
 '%25s exis
@@ -1010,16 +1010,17 @@
 fname, )
+)
 %0A    num
@@ -1043,17 +1043,19 @@
 xception
-,
+ as
  e:%0A    
@@ -1059,17 +1059,17 @@
    print
- 
+(
 'error d
@@ -1090,16 +1090,17 @@
  pdf_url
+)
 %0A    pri
@@ -1101,18 +1101,19 @@
    print
- e
+(e)
 %0A  %0A  pr
@@ -1115,17 +1115,17 @@
 %0A  print
- 
+(
 '%25d/%25d o
@@ -1171,16 +1171,17 @@
 len(db))
+)
 %0A  %0Aprin
@@ -1181,17 +1181,17 @@
   %0Aprint
- 
+(
 'final n
@@ -1248,9 +1248,11 @@
 len(db))
+)%0A
 %0A

converted download_pdfs to py3