from datetime import date, datetime, timedelta
import pytest
import requests
from web_test_base import *


class TestIATIDatastore(WebTestBase):
    requests_to_load = {
        'Datastore Homepage': {
            'url': 'http://datastore.iatistandard.org/'
        },
        'API - Activities Updated since 2 days ago': {
            'url': 'http://datastore.iatistandard.org/api/1/access/activity.xml?limit=0&last-updated-datetime__gt=' + str(date.today() - timedelta(days=2)),
            'min_response_size': 295
        },
        'API - Activities Updated since 3 days ago': {
            'url': 'http://datastore.iatistandard.org/api/1/access/activity.xml?limit=0&last-updated-datetime__gt=' + str(date.today() - timedelta(days=3)),
            'min_response_size': 295
        },
        'Datastore download: csv': {
            'url': 'http://datastore.iatistandard.org/api/1/access/activity.csv'
        },
        'Datastore download: xml': {
            'url': 'http://datastore.iatistandard.org/api/1/access/activity.xml'
        },
        'Datastore download: json': {
            'url': 'http://datastore.iatistandard.org/api/1/access/activity.json'
        },
        'Datastore list of datasets': {
            'url': 'http://datastore.iatistandard.org/api/1/about/dataset'
        }
    }

    @pytest.mark.parametrize("target_request", ["Datastore Homepage"])
    def test_contains_links(self, target_request):
        """Test that each page contains links to the defined URLs."""
        loaded_request = self.loaded_request_from_test_name(target_request)

        result = utility.get_links_from_page(loaded_request)

        assert "http://iatiregistry.org/" in result

    @pytest.mark.parametrize("target_request", ["API - Activities Updated since 2 days ago", "API - Activities Updated since 3 days ago"])
    def test_recent_activities(self, target_request):
        """Test that the datastore API knows of activities updated in the past few days."""
        req = self.loaded_request_from_test_name(target_request)
        xpath = '//result/iati-activities/query/total-count'

        result = utility.get_single_int_from_xpath(req, xpath)

        assert result > 0

    @pytest.mark.parametrize("content_type", ["application/xml", "application/json", "text/csv"])
    def test_api_output_filetype(self, content_type):
        """
        Test that API calls return data in the expected filetypes.

        The test is conducted based upon the data returned in the request
        headers["content-type"]. For example, 'application/xml; charset=utf-8'
        """
        file_extenstion = content_type.split("/")[1]
        loaded_request = self.loaded_request_from_test_name("Datastore download: {}".format(file_extenstion))

        result = loaded_request.headers["content-type"]

        assert result.startswith(content_type)

    @pytest.mark.slow
    @pytest.mark.parametrize("target_request", ["Datastore list of datasets"])
    def test_last_successful_fetch(self, target_request):
        """Test that the datastore has successfully fetched data within the expected time frame."""
        loaded_request = self.loaded_request_from_test_name(target_request)
        successful_fetch_dates = list()
        json_dataset_list = loaded_request.json()
        for dataset in json_dataset_list['datasets']:
            if dataset != None:
                dataset_url = '/'.join([loaded_request.url, dataset])
                data = requests.get(dataset_url).json()
                try:
                    if (data['resources'][0]['last_successful_fetch'] is not None) or (data['resources'] != list()):
                        successful_fetch_dates.append(datetime.strptime(data['resources'][0]['last_successful_fetch'][:10], '%Y-%m-%d'))
                except (TypeError, IndexError, ValueError):
                    pass
        assert max(successful_fetch_dates) >= (datetime.now() - timedelta(days=2))

@@ -1284,16 +1284,22 @@
 /dataset
+s/nest
 '%0A      
@@ -2861,24 +2861,26 @@
 t_type)%0A%0A   
+ #
  @pytest.mar
@@ -2886,16 +2886,16 @@
 rk.slow%0A
-
     @pyt
@@ -3259,21 +3259,17 @@
 _dataset
-_list
+s
  = loade
@@ -3297,22 +3297,26 @@
     
-for 
+list_of_
 dataset
- in
+s =
  jso
@@ -3328,13 +3328,9 @@
 aset
-_list
+s
 %5B'da
@@ -3337,17 +3337,17 @@
 tasets'%5D
-:
+%0A
 %0A       
@@ -3347,22 +3347,19 @@
         
-    if
+for
  dataset
@@ -3363,86 +3363,28 @@
 set 
-!= None:%0A                dataset_url = '/'.join(%5Bloaded_request.url, 
+in list_of_
 dataset
-%5D)
+s:
 %0A   
@@ -3396,49 +3396,34 @@
     
-    data = requests.get(dataset_url).json
+resources = dataset.values
 ()%0A 
@@ -3437,15 +3437,33 @@
     
- 
-   try
+for resource in resources
 :%0A  
@@ -3480,30 +3480,23 @@
     
- 
-   if (data%5B'
+date = 
 resource
 s'%5D%5B
@@ -3483,38 +3483,32 @@
  date = resource
-s'%5D%5B0%5D
 %5B'last_successfu
@@ -3520,60 +3520,46 @@
 ch'%5D
- is not None) or (data%5B'resources'%5D != list()):%0A    
+%0A                if date is not None:%0A
     
@@ -3629,154 +3629,28 @@
 (dat
-a%5B'resources'%5D%5B0%5D%5B'last_successful_fetch'%5D%5B:10%5D, '%25Y-%25m-%25d'))%0A                except (TypeError, IndexError, ValueError):%0A                    pass
+e%5B:10%5D, '%25Y-%25m-%25d'))
 %0A   
@@ -3666,10 +3666,10 @@
 rt m
-ax
+in
 (suc

Remove pdb and change test URL to live URL