@@ -0,0 +1,3016 @@
+from pydispatch import dispatcher%0A%0Afrom collections import defaultdict%0Afrom cookielib import CookieJar%0A%0Afrom scrapy.core import signals%0Afrom scrapy.utils.misc import dict_updatedefault%0Afrom scrapy import log%0A%0Aclass CookiesMiddleware(object):%0A    %22%22%22This middleware enables working with sites that need cookies%22%22%22%0A%0A    def __init__(self):%0A        self.cookies = defaultdict(self.defaultdict_factory)%0A        dispatcher.connect(self.domain_closed, signals.domain_closed)%0A%0A    def process_request(self, request, spider):%0A        if request.meta.get('dont_merge_cookies', False):%0A            return%0A        jar = self.cookies%5Bspider.domain_name%5D%0A        wreq = _WrappedRequest(request)%0A%0A        for key, value in request.cookies.items():%0A            jar.set_cookie%0A%0A%0A        jar.add_cookie_header(wreq)%0A        print request.url.netloc, '    Cookie: ', request.headers.get('Cookie'), request.cookies%0A%0A    def process_response(self, request, response, spider):%0A        print request.url.netloc, 'Set-Cookie:', response.headers.get('Set-Cookie'), request.cookies%0A        wreq = _WrappedRequest(request)%0A        wrsp = _WrappedResponse(response)%0A        jar = self.cookies%5Bspider.domain_name%5D%0A        jar.extract_cookies(wrsp, wreq)%0A        return response%0A%0A    def domain_closed(self, domain):%0A        if domain in self.cookies:%0A            del self.cookies%5Bdomain%5D%0A%0A    def defaultdict_factory(self):%0A        return CookieJar()%0A%0A%0Aclass _WrappedRequest(object):%0A    %22%22%22Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class%0A%0A    see http://docs.python.org/library/urllib2.html#urllib2.Request%0A    %22%22%22%0A%0A    def __init__(self, request):%0A        self.request = request%0A%0A    def get_full_url(self):%0A        return self.request.url%0A%0A    def get_host(self):%0A        return self.request.url.netloc%0A%0A    def get_type(self):%0A        return self.request.url.scheme%0A%0A    def is_unverifiable(self):%0A        %22%22%22Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.%0A%0A        It defaults to False. An unverifiable request is one whose URL the user did not have the%0A        option to approve. For example, if the request is for an image in an%0A        HTML document, and the user had no option to approve the automatic%0A        fetching of the image, this should be true.%0A        %22%22%22%0A        return self.request.meta.get('is_unverifiable', False)%0A%0A    def get_origin_req_host(self):%0A        return self.request.hostname%0A%0A    def has_header(self, name):%0A        return name in self.request.headers%0A%0A    def header_items(self):%0A        return self.request.headers.items()%0A%0A    def add_unredirected_header(self, name, value):%0A        # XXX: review please, not sure how to handle this%0A        self.request.headers%5Bname%5D = value%0A%0A%0Aclass _WrappedResponse(object):%0A%0A    def __init__(self, response):%0A        self.response = response%0A%0A    def info(self):%0A        return self%0A%0A    def getheaders(self, name):%0A        return self.response.headers.getlist(name)%0A

add new cookie middleware to experimental