import logging

from google.appengine.dist import use_library
use_library('django', '1.2')

appstats_RECORD_FRACTION = .1

class BlockingMiddleware(object):
    def __init__(self, app):
        self._wrapped_app = app

    def __call__(self, environ, start_response):
        user_agent = environ.get('HTTP_USER_AGENT', '')
        # Scraper running on EC2 (IP addresses 50.18.2.106 and 50.18.73.38)
        # that's requesting lots of Tweet Digest pages.
        if user_agent == 'Python-urllib/2.7':
            logging.info('Blocked request')
            start_response('403 Forbidden', [('Content-type','text/plain')])
            return ['']

        # Google Reader and iGoogle are dead, yet their crawler still lives on.
        # (and generates ~1,200 requests per day). Block it, since presumably no
        # one is actually looking at the resuts. Make the block be a 200
        # response, so that it gets cached by App Engine's edge cache (and
        # hopefully Trawler too).
        if user_agent.startswith('Feedfetcher-Google;'):
            logging.info('Blocked Feedfetcher request')
            start_response('200 OK', [
                ('Content-type','text/xml; charset=UTF-8'),
                ('Last-Modified', 'Tue, 2 Jul 2013 00:00:00 GMT'),
                ('Expires', 'Sun, 2 Jul 2023 00:00:00 GMT'),
                ('Cache-Control', 'public, max-age=315570000'),
            ])
            return ['''<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
  <id>tag:streamspigot.com,2013:/feedfetcher-go-away</id>
  <title>Stream Spigot</title>
  <entry>
    <id>tag:streamspigot.com,2013:/feedfetcher-go-away</id>
    <title type="text">Feedfetcher, why are you still crawling?</title>
    <content type="xhtml">
      <div xmlns="http://www.w3.org/1999/xhtml">
        Since <a href="http://googlereader.blogspot.com/2013/07/a-final-farewell.html">Reader</a> and <a href="https://support.google.com/websearch/answer/2664197?hl=en">iGoogle</a> are shut down, why is <a href="http://www.google.com/feedfetcher.html">Feedfetcher</a> still crawling?
      </div>
    </content>
  </entry>
</feed>
''']
        return self._wrapped_app(environ, start_response)

def webapp_add_wsgi_middleware(app):
    from google.appengine.ext.appstats import recording
    app = recording.appstats_wsgi_middleware(app)

    app = BlockingMiddleware(app)

    return app

@@ -854,141 +854,8 @@
 uts.
- Make the block be a 200%0A        # response, so that it gets cached by App Engine's edge cache (and%0A        # hopefully Trawler too).
 %0A   
@@ -996,1034 +996,81 @@
 se('
-200 OK', %5B%0A                ('Content-type','text/xml; charset=UTF-8'),%0A                ('Last-Modified', 'Tue, 2 Jul 2013 00:00:00 GMT'),%0A                ('Expires', 'Sun, 2 Jul 2023 00:00:00 GMT'),%0A                ('Cache-Control', 'public, max-age=315570000'),%0A            %5D)%0A            return %5B'''%3C?xml version=%221.0%22 encoding=%22utf-8%22?%3E%0A%3Cfeed xmlns=%22http://www.w3.org/2005/Atom%22%3E%0A  %3Cid%3Etag:streamspigot.com,2013:/feedfetcher-go-away%3C/id%3E%0A  %3Ctitle%3EStream Spigot%3C/title%3E%0A  %3Centry%3E%0A    %3Cid%3Etag:streamspigot.com,2013:/feedfetcher-go-away%3C/id%3E%0A    %3Ctitle type=%22text%22%3EFeedfetcher, why are you still crawling?%3C/title%3E%0A    %3Ccontent type=%22xhtml%22%3E%0A      %3Cdiv xmlns=%22http://www.w3.org/1999/xhtml%22%3E%0A        Since %3Ca href=%22http://googlereader.blogspot.com/2013/07/a-final-farewell.html%22%3EReader%3C/a%3E and %3Ca href=%22https://support.google.com/websearch/answer/2664197?hl=en%22%3EiGoogle%3C/a%3E are shut down, why is %3Ca href=%22http://www.google.com/feedfetcher.html%22%3EFeedfetcher%3C/a%3E still crawling?%0A      %3C/div%3E%0A    %3C/content%3E%0A  %3C/entry%3E%0A%3C/feed%3E%0A'''%5D
+403 Forbidden', %5B('Content-type','text/plain')%5D)%0A            return %5B''%5D%0A
 %0A   

Revert "Give a 200 response to Feedfetcher so that it can be cached."