import string

from ..common.Harvester import Harvester
from oaipmh.client import Client
from oaipmh.metadata import MetadataRegistry, oai_dc_reader


class CiteseerxHarvester(Harvester):
    
    OAI_PHM_URL = 'http://citeseerx.ist.psu.edu/oai2'
    
    def __init__(self):
        super(CiteseerxHarvester, self).__init__('citeseerx')
    
    def harvest(self):
        registry = MetadataRegistry()
        registry.registerReader('oai_dc', oai_dc_reader)
        client = Client(self.OAI_PHM_URL, registry)
        #harvestStart = datetime.strptime("2014-06-17T23:59:59Z", "%Y-%m-%dT%H:%M:%SZ")
        for record in client.listRecords(metadataPrefix='oai_dc'):
            header = record[0];
            metadata = record[1];
    
            title = ''
            date = ''
            publisher = ''
            abstract = ''
            source = ''
    
            if metadata['title']:
                title = metadata['title'][0]
            if metadata['date']:
                date = metadata['date'][-1]
                '''
                <dc:date>2009-04-19</dc:date>
                <dc:date>2007-11-19</dc:date>
                <dc:date>1998</dc:date>
                '''
                if len(date) != 4:
                    date = ''
            if metadata['description']:
                abstract = metadata['description'][0]
            if metadata['publisher']:
                publisher = metadata['publisher'][0]
            #if metadata['source']:
            #    source = metadata['source'][0]
            
            # <identifier>oai:CiteSeerX.psu:10.1.1.1.1519</identifier>
            citeseerx_id = string.replace(header.identifier(), 'oai:CiteSeerX.psu:', '')
            source = "http://citeseerx.ist.psu.edu/viewdoc/download?doi="+citeseerx_id+"&amp;rep=rep1&amp;type=pdf"            
            
            self.open_split_file()            
            self.parse_publication(
                authors=metadata['creator'],
                title=title,
                date=date,
                publisher=publisher,
                abstract=abstract,
                source=source,
                citeseerx_id=citeseerx_id)
            
            if self.check_stop_harvest():
                break
            

        self.stop_harvest()
@@ -241,24 +241,230 @@
 /oai2'%0A    %0A
+    FIELD_MAPPING = %7B%0A        'title': 'title',%0A        #'creator': 'authors',%0A        'description': 'abstract',%0A        'publisher': 'publisher',%0A        'rights': 'copyright'%0A        #subject%0A    %7D%0A    %0A
     def __in
@@ -933,33 +933,32 @@
 ord%5B1%5D;%0A    
-%0A
         
     title = 
@@ -949,71 +949,37 @@
     
-    title = ''%0A            date = ''%0A            publisher = ''
+%0A            result_entry = %7B
 %0A   
@@ -991,21 +991,22 @@
     
-abstract = ''
+    'urls': %5B%5D
 %0A   
@@ -1018,19 +1018,9 @@
     
-source = ''
+%7D
 %0A   
@@ -1046,21 +1046,23 @@
 tadata%5B'
-title
+creator
 '%5D:%0A    
@@ -1073,21 +1073,39 @@
         
-title
+result_entry%5B'authors'%5D
  = metad
@@ -1113,17 +1113,16 @@
 ta%5B'
-title'%5D%5B0
+creator'
 %5D%0A  
@@ -1401,9 +1401,9 @@
 te) 
-!
+=
 = 4:
@@ -1427,17 +1427,35 @@
     
+result_entry%5B'
 date
+'%5D
  = 
-''
+date
 %0A   
@@ -1476,27 +1476,22 @@
 tadata%5B'
-description
+source
 '%5D:%0A    
@@ -1502,24 +1502,19 @@
         
-abstract
+url
  = metad
@@ -1522,25 +1522,24 @@
 ta%5B'
-description
+source
 '%5D%5B0%5D%0A
+    
     
@@ -1559,25 +1559,22 @@
 tadata%5B'
-publisher
+format
 '%5D:%0A    
@@ -1589,58 +1589,244 @@
     
-publisher = metadata%5B'publisher'%5D%5B0%5D%0A            #
+    url = %7B%0A                        'value': url,%0A                        'type': metadata%5B'format'%5D%5B0%5D%0A                    %7D%0A                result_entry%5B'urls'%5D.append(url)%0A            for field in self.FIELD_MAPPING:%0A                
 if m
@@ -1829,32 +1829,29 @@
 if metadata%5B
-'source'
+field
 %5D:%0A         
@@ -1857,19 +1857,55 @@
     
-#
+  
     
-source
+  result_entry%5Bself.FIELD_MAPPING%5Bfield%5D%5D
  = m
@@ -1908,32 +1908,29 @@
  = metadata%5B
-'source'
+field
 %5D%5B0%5D%0A       
@@ -2010,32 +2010,46 @@
 er%3E%0A            
+result_entry%5B'
 citeseerx_id = s
@@ -2044,16 +2044,18 @@
 seerx_id
+'%5D
  = strin
@@ -2127,18 +2127,64 @@
     
-source = %22
+result_entry%5B'urls'%5D.append(%7B%0A                'value': '
 http
@@ -2233,24 +2233,10 @@
 doi=
-%22+citeseerx_id+%22
+%7B%7D
 &amp
@@ -2261,363 +2261,213 @@
 =pdf
-%22            %0A            %0A            self.open_split_file()            %0A            self.parse_publication(%0A                authors=metadata%5B'creator'%5D,%0A                title=title,%0A                date=date,%0A                publisher=publisher,%0A                abstract=abstract,%0A                source=source,%0A                citeseerx_id=citeseerx_id
+'.format(result_entry%5B'citeseerx_id'%5D),%0A                'type': 'application/pdf'%0A            %7D)%0A            %0A            self.open_split_file()            %0A            self.parse_publication2(result_entry
 )%0A  

Use new format in citeseer harvester