from scrapy.utils.reqser import request_to_dict, request_from_dict

from . import picklecompat


class Base(object):
    """Per-spider base queue class"""

    def __init__(self, server, spider, key, serializer=None):
        """Initialize per-spider redis queue.

        Parameters
        ----------
        server : StrictRedis
            Redis client instance.
        spider : Spider
            Scrapy spider instance.
        key: str
            Redis key where to put and get messages.
        serializer : object
            Serializer object with ``loads`` and ``dumps`` methods.

        """
        if serializer is None:
            # Backward compatibility.
            # TODO: deprecate pickle.
            serializer = picklecompat
        if not hasattr(serializer, 'loads'):
            raise TypeError("serializer does not implement 'loads' function: %r"
                            % serializer)
        if not hasattr(serializer, 'dumps'):
            raise TypeError("serializer '%s' does not implement 'dumps' function: %r"
                            % serializer)

        self.server = server
        self.spider = spider
        self.key = key % {'spider': spider.name}
        self.serializer = serializer

    def _encode_request(self, request):
        """Encode a request object"""
        obj = request_to_dict(request, self.spider)
        return self.serializer.dumps(obj)

    def _decode_request(self, encoded_request):
        """Decode an request previously encoded"""
        obj = self.serializer.loads(encoded_request)
        return request_from_dict(obj, self.spider)

    def __len__(self):
        """Return the length of the queue"""
        raise NotImplementedError

    def push(self, request):
        """Push a request"""
        raise NotImplementedError

    def pop(self, timeout=0):
        """Pop a request"""
        raise NotImplementedError

    def clear(self):
        """Clear queue/stack"""
        self.server.delete(self.key)


class FifoQueue(Base):
    """Per-spider FIFO queue"""

    def __len__(self):
        """Return the length of the queue"""
        return self.server.llen(self.key)

    def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, self._encode_request(request))

    def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            data = self.server.brpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            data = self.server.rpop(self.key)
        if data:
            return self._decode_request(data)


class PriorityQueue(Base):
    """Per-spider priority queue abstraction using redis' sorted set"""

    def __len__(self):
        """Return the length of the queue"""
        return self.server.zcard(self.key)

    def push(self, request):
        """Push a request"""
        data = self._encode_request(request)
        score = -request.priority
        # We don't use zadd method as the order of arguments change depending on
        # whether the class is Redis or StrictRedis, and the option of using
        # kwargs only accepts strings, not bytes.
        self.server.execute_command('ZADD', self.key, score, data)

    def pop(self, timeout=0):
        """
        Pop a request
        timeout not support in this queue class
        """
        # use atomic range/remove using multi/exec
        pipe = self.server.pipeline()
        pipe.multi()
        pipe.zrange(self.key, 0, 0).zremrangebyrank(self.key, 0, 0)
        results, count = pipe.execute()
        if results:
            return self._decode_request(results[0])


class LifoQueue(Base):
    """Per-spider LIFO queue."""

    def __len__(self):
        """Return the length of the stack"""
        return self.server.llen(self.key)

    def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, self._encode_request(request))

    def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            data = self.server.blpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            data = self.server.lpop(self.key)

        if data:
            return self._decode_request(data)

class SpiderSimpleQueue(Base):
    """ url + callback """

    def __len__(self):
        """Return the length of the queue"""
        return self.server.llen(self.key)

    def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, request.url[16:])

    def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            url = self.server.brpop(self.key, timeout=timeout)
            if isinstance(url, tuple):
                url = url[1]
        else:
            url = self.server.rpop(self.key)
        if url:
            try:
                if "/followers" in url or "/followees" in url:
                    cb = getattr(self.spider, "parse_relation")
                elif "/answers" in url:
                    cb = getattr(self.spider, "parse_answers")
                elif "/asks" in url:
                    cb = getattr(self.spider, "parse_question")
                elif "/articles" in url:
                    cb = getattr(self.spider, "parse_articles")
                elif "/include=locations" in url:
                    cb = getattr(self.spider, "parse")
                else:
                    raise ValueError("Method not found in: %s( URL:%s )" % (self.spider, url))
                return Request(url="https://weibo.cn%s" % url, callback=cb)
            except AttributeError:
                raise ValueError("Method not found in: %s( URL:%s )" % (self.spider, url))


# TODO: Deprecate the use of these names.
SpiderQueue = FifoQueue
SpiderStack = LifoQueue
SpiderPriorityQueue = PriorityQueue
@@ -4289,1458 +4289,8 @@
 a)%0A%0A
-class SpiderSimpleQueue(Base):%0A    %22%22%22 url + callback %22%22%22%0A%0A    def __len__(self):%0A        %22%22%22Return the length of the queue%22%22%22%0A        return self.server.llen(self.key)%0A%0A    def push(self, request):%0A        %22%22%22Push a request%22%22%22%0A        self.server.lpush(self.key, request.url%5B16:%5D)%0A%0A    def pop(self, timeout=0):%0A        %22%22%22Pop a request%22%22%22%0A        if timeout %3E 0:%0A            url = self.server.brpop(self.key, timeout=timeout)%0A            if isinstance(url, tuple):%0A                url = url%5B1%5D%0A        else:%0A            url = self.server.rpop(self.key)%0A        if url:%0A            try:%0A                if %22/followers%22 in url or %22/followees%22 in url:%0A                    cb = getattr(self.spider, %22parse_relation%22)%0A                elif %22/answers%22 in url:%0A                    cb = getattr(self.spider, %22parse_answers%22)%0A                elif %22/asks%22 in url:%0A                    cb = getattr(self.spider, %22parse_question%22)%0A                elif %22/articles%22 in url:%0A                    cb = getattr(self.spider, %22parse_articles%22)%0A                elif %22/include=locations%22 in url:%0A                    cb = getattr(self.spider, %22parse%22)%0A                else:%0A                    raise ValueError(%22Method not found in: %25s( URL:%25s )%22 %25 (self.spider, url))%0A                return Request(url=%22https://weibo.cn%25s%22 %25 url, callback=cb)%0A            except AttributeError:%0A                raise ValueError(%22Method not found in: %25s( URL:%25s )%22 %25 (self.spider, url))%0A%0A%0A
 # TO
@@ -4371,24 +4371,24 @@
 = LifoQueue%0A
-
 SpiderPriori
@@ -4398,20 +4398,21 @@
 ueue = PriorityQueue
+%0A

Update queue.py