"""
Offsite Spider Middleware

See documentation in docs/ref/spider-middleware.rst
"""

from scrapy.http import Request
from scrapy.utils.url import url_is_from_spider

class OffsiteMiddleware(object):

    def process_spider_output(self, response, result, spider):
        return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))

@@ -326,22 +326,85 @@
 est) or 
+%5C%0A            self.is_url_from_spider(x.url, spider))%0A%0A    def is_
 url
-_is
 _from_sp
@@ -400,32 +400,36 @@
 url_from_spider(
-x.
+self, 
 url, spider))%0A
@@ -414,22 +414,126 @@
 r(self, url, spider)
+:%0A        # XXX: should we ask this to the spider manager?%0A        return url_is_from_spider(url, spider
 )%0A

isolate policy of urls belonging to spiders into a separate method