"""
Open Science Framework harvester of public projects for the SHARE Notification Service

Example API query: https://osf.io/api/v1/search/
https://staging.osf.io/api/v1/search/?q=category:registration%20AND%20date_created:[2015-01-01%20TO%202015-03-10]&size=1000
https://osf.io/api/v1/search/?q=category:registration%20AND%20NOT%20title=test%20AND%20NOT%20title=%22Test%20Project%22
"""

from __future__ import unicode_literals

import json
import logging
from datetime import date, timedelta

from nameparser import HumanName

from scrapi import requests
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties, date_formatter

logger = logging.getLogger(__name__)


def process_contributors(authors):

    contributor_list = []
    for person in authors:
        name = HumanName(person['fullname'])
        contributor = {
            'name': person['fullname'],
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
        }
        contributor_list.append(contributor)

    return contributor_list


def process_null(entry):
    if entry is None:
        return ''
    else:
        return entry


def process_tags(entry):
    if isinstance(entry, list):
        return entry
    else:
        return [entry]


class OSFHarvester(JSONHarvester):
    short_name = 'osf'
    long_name = 'Open Science Framework'
    url = 'http://osf.io/api/v1/search/'
    count = 0

    # Only registrations that aren't just the word "test" or "test project"
    URL = 'https://osf.io/api/v1/search/?q=category:registration ' +\
          ' AND registered_date:[{} TO {}]' +\
          ' AND NOT title=test AND NOT title="Test Project"&size=1000'

    @property
    def schema(self):
        return {
            'contributors': ('/contributors', process_contributors),
            'title': ('/title', process_null),
            'providerUpdatedDateTime': ('/date_registered', date_formatter),
            'description': ('/description', process_null),
            'uris': {
                'canonicalUri': ('/url', lambda x: 'http://osf.io' + x),
            },
            'tags': ('/tags', process_tags),
            'otherProperties': build_properties(
                ('parent_title', '/parent_title'),
                ('category', '/category'),
                ('wiki_link', '/wiki_link'),
                ('is_component', '/is_component'),
                ('is_registration', '/is_registration'),
                ('parent_url', '/parent_url'),
                ('journal Id', '/journal Id')
            )
        }

    def harvest(self, start_date=None, end_date=None):
        # Always harvest a 2 day period starting 2 days back to honor time given
        # to contributors to cancel a public registration
        start_date = start_date or date.today() - timedelta(4)
        end_date = end_date or date.today() - timedelta(2)

        search_url = self.URL.format(start_date.isoformat(), end_date.isoformat())
        records = self.get_records(search_url)

        record_list = []
        for record in records:
            doc_id = record['url'].replace('/', '')

            record_list.append(
                RawDocument(
                    {
                        'doc': json.dumps(record),
                        'source': self.short_name,
                        'docID': doc_id,
                        'filetype': 'json'
                    }
                )
            )

        return record_list

    def get_records(self, search_url):
        records = requests.get(search_url)

        try:
            total = int(records.json()['counts']['registration'])
        except KeyError:
            return []

        from_arg = 0
        all_records = []
        while len(all_records) < total:
            record_list = records.json()['results']

            for record in record_list:
                all_records.append(record)

            from_arg += 1000
            records = requests.get(search_url + '&from={}'.format(str(from_arg)), throttle=10)

        return all_records

@@ -493,42 +493,8 @@
 ta%0A%0A
-from nameparser import HumanName%0A%0A
 from
@@ -634,16 +634,71 @@
  import 
+(%0A    compose,%0A    parse_name,%0A    coerce_to_list,%0A    
 build_pr
@@ -706,20 +706,28 @@
 perties,
- dat
+%0A    datetim
 e_format
@@ -729,16 +729,18 @@
 ormatter
+%0A)
 %0A%0Alogger
@@ -771,16 +771,59 @@
 ame__)%0A%0A
+url_from_guid = 'https://osf.io%7B%7D'.format%0A%0A
 %0Adef pro
@@ -916,21 +916,29 @@
     
-name = HumanN
+contributor = parse_n
 ame(
@@ -980,464 +980,125 @@
 utor
- = %7B%0A            'name': person%5B'fullname'%5D,%0A            'givenName': name.first,%0A            'additionalName': name.middle,%0A            'familyName': name.last,%0A        %7D%0A        contributor_list.append(contributor)%0A%0A    return contributor_list%0A%0A%0Adef process_null(entry):%0A    if entry is None:%0A        return ''%0A    else:%0A        return entry%0A%0A%0Adef process_tags(entry):%0A    if isinstance(entry, list):%0A        return entry%0A    else:%0A        return %5Bentry%5D
+%5B'sameAs'%5D = %5Burl_from_guid(person%5B'url'%5D)%5D%0A        contributor_list.append(contributor)%0A%0A    return contributor_list
 %0A%0A%0Ac
@@ -1670,28 +1670,33 @@
 title', 
-process_null
+lambda x: x or ''
 ),%0A     
@@ -1754,16 +1754,20 @@
 d', date
+time
 _formatt
@@ -1798,17 +1798,16 @@
 ption': 
-(
 '/descri
@@ -1817,23 +1817,8 @@
 on',
- process_null),
 %0A   
@@ -1881,39 +1881,104 @@
 l', 
-lambda x: 'http://osf.io' + x),
+url_from_guid),%0A                'providerUris': ('/url', compose(coerce_to_list, url_from_guid))
 %0A   
@@ -2013,17 +2013,16 @@
 s': 
-(
 '/tags',
  pro
@@ -2021,23 +2021,8 @@
 gs',
- process_tags),
 %0A   
@@ -3428,25 +3428,8 @@
 l)%0A%0A
-        try:%0A    
     
@@ -3489,55 +3489,8 @@
 n'%5D)
-%0A        except KeyError:%0A            return %5B%5D
 %0A%0A  

Bring OSF up to date with the new best practices for data output