import sys
import os

path = os.path.abspath('.')
sys.path.append(path)

import downloader as d

good_urls = ['http://yle.fi/uutiset/tv-uutisten_politiikantoimittaja_timo_kuparinen_kuollut/9167149']
bad_urls = ['http://example.com', 'http://www.example.org']


class TestParser:

    @classmethod
    def setup_class(self):

        self.raw_path = 'test-data-raw/'
        self.data_path = 'test-data/'
        self.errors = 'test-errors.log'

        errors = open( self.errors, 'w')

        for f in [ self.raw_path, self.data_path ]:
            if not os.path.exists( f ):
                os.makedirs( f )

        for url in good_urls:
            d.download( url , self.raw_path, errors )

        for url in bad_urls:
            d.download( url , self.raw_path, errors )

        self.collected = d.resort_pickles( self.raw_path )

    def test_downloaded_files_exists(self):
        assert len( os.listdir( self.raw_path ) ) == len( good_urls )

    def test_errors_logged(self):
        assert len( open( self.errors ).readlines() ) == len( bad_urls )

    def test_file_contents_ok(self):

        import pickle

        keys = ['url', 'http', 'categories', 'datetime_list', 'author', 'title', 'ingress', 'text', 'images', 'captions']

        for d in os.listdir( self.raw_path ):
            d = pickle.load( open( self.raw_path + d ) )
            for key in keys:
                assert key in d

    def test_pickles_sorted_correctly(self):
        count = sum( map( lambda x: len( x ), self.collected.values() ) )
        assert count == len( good_urls )

    def test_pickles_sorted_keys(self):

        print self.collected.keys()

        assert 'yle' in ''.join( self.collected.keys() )
        assert '2016' in ''.join( self.collected.keys() )
        assert '9' in ''.join( self.collected.keys() )

@@ -95,32 +95,37 @@
 d%0A%0Agood_urls = %5B
+%0A    
 'http://yle.fi/u
@@ -195,16 +195,137 @@
 9167149'
+,%0A    'http://www.hs.fi/kotimaa/art-2000005007566.html',%0A    'http://www.iltasanomat.fi/musiikki/art-2000005008209.html'%0A
 %5D%0Abad_ur
@@ -1747,36 +1747,27 @@
     
-print self.collected.keys()%0A
+## check file names
 %0A   
@@ -1812,32 +1812,171 @@
 lected.keys() )%0A
+        assert 'hs' in ''.join( self.collected.keys() )%0A        assert 'iltasanomat' in ''.join( self.collected.keys() )%0A%0A        ## years%0A
         assert '
@@ -2008,32 +2008,50 @@
 llected.keys() )
+%0A%0A        ## month
 %0A        assert 
@@ -2070,28 +2070,84 @@
 in( self.collected.keys() )%0A
+        assert '12' in ''.join( self.collected.keys() )%0A

Check more urls