from contextlib import closing
from operator import itemgetter
from argparse import ArgumentParser
from csv import DictWriter, DictReader
import sqlite3
import json
import re

FIELDS = ['Ateneo', 'Facoltà', 'Fascia', 'Genere', 'S.C.',
          'Servizio prestato in altro ateneo', 'Struttura di afferenza',
          'author', 'identifier', 'eid', 'title', 'aggregationType',
          'citedby-count', 'publicationName', 'isbn', 'issn', 'volume',
          'issueIdentifier', 'pageRange', 'pageNum', 'coverDate',
          'coverDisplayDate', 'doi', 'numAuthors']

QUERY = 'SELECT entries FROM articles WHERE author = ?'

def pagenum(pageRange):
    try:
        page = list(map(int, pageRange.split('-')))
        return 1 if len(page) == 1 else page[1] - page[0]
    except:
        return None

def process(entry):
    for key, value in list(entry.items()):
        if ':' in key:
            del entry[key]
            key = key.partition(':')[2]
            entry[key] = value

    match = re.match('Author list of (\d+)', entry.get('message', ''))
    if match: entry['numAuthors'] = int(match.group(1))
    else: entry['numAuthors'] = len(entry.get('author', [])) or None

    entry['author'] = entry['Cognome e Nome']
    entry['pageNum'] = pagenum(entry.get('pageRange', None))
    return entry


def mergedicts(*dicts):
    return {k:v for d in dicts for k,v in d.items()}


if __name__ == '__main__':

    parser = ArgumentParser('convert scopus db to csv')
    parser.add_argument('INFILE', help='input csv file')
    parser.add_argument('DBFILE', help='database file')
    parser.add_argument('OUTFILE', help='output csv file')
    args = parser.parse_args()

    with open(args.INFILE) as infile, open(args.OUTFILE, 'w') as outfile:
        csvreader = DictReader(infile)
        authors = [(row['Cognome e Nome'], row) for row in csvreader]
        authors.sort(key=itemgetter(0))

        csvwriter = DictWriter(outfile, FIELDS, extrasaction='ignore')
        csvwriter.writeheader()

        with sqlite3.connect(args.DBFILE) as connection:
            with closing(connection.cursor()) as cursor:
                for author, authordata in authors:
                    entries = cursor.execute(QUERY, (author,)).fetchall()
                    if not entries:
                        print('Empty entry added for %s' % author)
                        authordata['eid'] = authordata['identifier'] = 0
                        csvwriter.writerow(process(authordata))
                    else:
                        inserted = set()
                        for entry in json.loads(entries[0][0]):
                            ID = entry.get('dc:identifier', '')
                            if ID in inserted:
                                print('Ignore duplicate (%s,%s)' % (author, ID))
                            else:
                                inserted.add(ID)
                                csvwriter.writerow(process(mergedicts(authordata, entry)))

@@ -56,16 +56,46 @@
 mgetter%0A
+from datetime import datetime%0A
 from arg
@@ -122,16 +122,16 @@
 tParser%0A
-
 from csv
@@ -1205,16 +1205,520 @@
 r None%0A%0A
+    # eid and identifier default to 0%0A    entry%5B'eid'%5D = entry.get('eid', 0)%0A    entry%5B'identifier'%5D = entry.get('identifier', 0)%0A%0A    # validate coverDate (or default to 1900-01-01)%0A    date = entry.get('coverDate', '')%0A    try:%0A        datesplit = list(map(int, date.split('-')))%0A        if len(datesplit) == 3 and datesplit%5B1%5D == 0:%0A            date = '%25d-%25d-%25s' %25 (datesplit%5B0%5D, datesplit%5B1%5D+1, datesplit%5B2%5D)%0A        datetime.strptime(date, '%25Y-%25m-%25d')%0A    except: entry%5B'coverDate'%5D = '1900-01-01'%0A%0A
     entr
@@ -2805,16 +2805,16 @@
 ntries:%0A
+
         
@@ -2876,81 +2876,8 @@
 or)%0A
-                        authordata%5B'eid'%5D = authordata%5B'identifier'%5D = 0%0A
     

validate and correct date