#!/usr/bin/env python

from datetime import datetime
import os
import requests

from pyquery import PyQuery

from app_config import get_secrets
from models import Story

SECRETS = get_secrets()

class SeamusScraper:
    def __init__(self):
        self.run_time = datetime.utcnow()

    def scrape_seamus(self, **kwargs):
        """
        Scrape!
        """
        print 'Scraping Seamus'
        print '---------------'

        if not kwargs:
            response = requests.get('http://api.npr.org/query', params={
                'date': 'current',
                'orgId': '1',
                'apiKey': SECRETS['NPR_API_KEY']})

            element = PyQuery(response.content, parser='xml')
        else:
            element = PyQuery(parser='xml', **kwargs)

        story_elements = element.find('story')
        stories = []

        for story_el in story_elements:
            story_el = PyQuery(story_el, parser='xml')
            print story_el.attr('id')
            stories.append(Story(story_el, self.run_time))

        return stories

    def write(self, db, stories):
        """
        Write to database
        """
        table = db['seamus']

        for story in stories:
            exists = table.find_one(id=story.id)

            if exists:
                continue

            row = story.serialize()
            table.insert(row)

@@ -186,16 +186,42 @@
 ecrets()
+%0ASEAMUS_API_PAGE_SIZE = 20
 %0A%0Aclass 
@@ -485,363 +485,1002 @@
     
-response = requests.get('http://api.npr.org/query', params=%7B%0A                'date': 'current',%0A                'orgId': '1',%0A                'apiKey': SECRETS%5B'NPR_API_KEY'%5D%7D)%0A%0A            element = PyQuery(response.content, parser='xml')%0A        else:%0A            element = PyQuery(parser='xml', **kwargs)%0A%0A        story_elements = element.find('story')
+stories = self._get_stories_from_api()%0A        else:%0A            element = PyQuery(parser='xml', **kwargs)%0A            story_elements = element.find('story')%0A            stories = self._extract_stories(story_elements)%0A%0A        return stories%0A%0A    def _get_stories_from_api(self):%0A        startNum = 1%0A        stories = %5B%5D%0A        while True:%0A            response = requests.get('http://api.npr.org/query', params=%7B%0A                'date': 'current',%0A                'orgId': '1',%0A                'apiKey': SECRETS%5B'NPR_API_KEY'%5D,%0A                'numResults': 20,%0A                'startNum': startNum,%0A            %7D)%0A%0A            element = PyQuery(response.content, parser='xml')%0A            story_elements = element.find('story')%0A%0A            if len(story_elements):%0A                stories += self._extract_stories(story_elements)%0A                startNum += SEAMUS_API_PAGE_SIZE%0A            else:%0A                break%0A%0A        return stories%0A%0A    def _extract_stories(self, story_elements):
 %0A   
@@ -1707,32 +1707,33 @@
 return stories%0A%0A
+%0A
     def write(se

handle pagination in seamus api, closes #74