@@ -0,0 +1,1413 @@
+import os%0Afrom google.cloud import datastore%0Aimport pytest%0Afrom scrapy import crawler%0Afrom scrapy.settings import Settings%0Afrom scraper import storage%0Afrom scraper.justice.spiders import acts%0A%0A%0A@pytest.fixture%0Adef crawler_settings() -%3E Settings:%0A    from scraper.justice import settings%0A    settings_ = Settings()%0A    settings_.setmodule(settings)%0A    settings_.set('ROBOTSTXT_OBEY', False)%0A    return settings_%0A%0A%0Adef test_crawler(datastore_client: datastore.Client, stor: storage.Storage, crawler_settings: Settings) -%3E None:%0A    test_home = os.path.join(os.path.dirname(__file__), 'justice/fixtures/acts_home.html')%0A    start_urls = %5B'file://' + test_home%5D%0A%0A    process = crawler.CrawlerProcess(crawler_settings)%0A    process.crawl(acts.ActsSpider, start_urls=start_urls)%0A    process.start()%0A    process.join()%0A%0A    query = datastore_client.query(kind='Act')%0A    result = sorted(list(query.fetch()), key=lambda x: (x%5B'code'%5D, x%5B'start'%5D))%0A    assert len(result) == 6%0A%0A    assert dict(result%5B0%5D) == %7B%0A        'code': 'A-1',%0A        'title': 'Access to Information Act',%0A        'start': '2015-07-09',%0A        'end': '2015-07-29',%0A        'raw_blob': 'acts/raw/A-1'%0A    %7D%0A    blob_text = stor.get_blob('acts/raw/A-1').download_to_string()%0A    assert len(blob_text) %3E 100%0A    assert '%3Cdiv class=%22info%22%3EVersion of document from 2015-07-09 to 2015-07-29:%3C/div%3E' in blob_text%0A%0A    assert result%5B2%5D%5B'code'%5D == 'B-1.01'%0A

add integration test