## Twitter API only provides the latest 3k statuses
## this script allows us to go deeper into Twitter history by using
## their website

import requests
import bs4
import re

match_id = '.*/([0-9]+)'

def _collect_ids( username , date ):

    url = 'https://mobile.twitter.com/search?src=typd&q=from%3AJyrkikasvi%20since%3A2011-01-01%20until%3A2011-04-01'

    r = requests.get( url )

    page = bs4.BeautifulSoup( r.text )

    ret = []

    for tweet in page.find( class_ ='timeline' ).children:

        if isinstance( tweet , bs4.Tag ):

        ## get tweet ID
            if 'href' in tweet.attrs:
                href = tweet['href']
                href = re.search( match_id, href )
                ret.append( (int)( href.group(1) ) )

    return ret

if __name__ == '__main__':
    print _collect_ids( 'matnel', 'x' )

@@ -167,16 +167,32 @@
 mport re
+%0Aimport datetime
 %0A%0Amatch_
@@ -245,24 +245,68 @@
  , date ):%0A%0A
+    date2 = date + datetime.timedelta( 1 )%0A%0A
     url = 'h
@@ -360,61 +360,91 @@
 m%253A
-Jyrkikasvi%2520since%253A2011-01-01%2520until%253A2011-04-01'
+' + username + '%2520since%253A' + date.isoformat() + '%2520until%253A' + date2.isoformat()
 %0A%0A  
@@ -635,16 +635,20 @@
         
+    
 ## get t
@@ -843,18 +843,330 @@
     
-return ret
+        ## TODO: check if count %3E 19; then need to parse more%0A%0A    return ret%0A%0Adef collect_tweets( username, since, until ):%0A%0A    date = since%0A%0A    tweets = %5B%5D%0A%0A    while date %3C= until:%0A%0A        tweets += _collect_ids( username, date )%0A        date = date + datetime.timedelta( 1 )%0A%0A        print date%0A%0A    return tweets%0A%0A
 %0A%0Aif
@@ -1200,25 +1200,29 @@
   print 
-_
+len( 
 collect_
 ids( 'ma
@@ -1217,25 +1217,88 @@
 ect_
-ids( 'matnel', 'x'
+tweets( 'Jyrkikasvi', datetime.date( 2011, 1, 1 ), datetime.date( 2011, 1, 31 ) )
  )%0A

Iterate over dates