@@ -0,0 +1,2275 @@
+from frontera.core import OverusedBuffer%0Afrom frontera.core.models import Request%0A%0A%0Ar1 = Request('http://www.example.com')%0Ar2 = Request('http://www.example.com/some/')%0Ar3 = Request('htttp://www.example.com/some/page/')%0Ar4 = Request('http://example.com')%0Ar5 = Request('http://example.com/some/page')%0Ar6 = Request('http://example1.com')%0A%0A%0Aclass TestOverusedBuffer(object):%0A%0A    requests = %5B%5D%0A    logs = %5B%5D%0A%0A    def get_func(self, max_n_requests, **kwargs):%0A        lst = %5B%5D%0A        for _ in range(max_n_requests):%0A            if self.requests:%0A                lst.append(self.requests.pop())%0A        return lst%0A%0A    def log_func(self, msg):%0A        self.logs.append(msg)%0A%0A    def test(self):%0A        ob = OverusedBuffer(self.get_func, self.log_func)%0A        self.requests = %5Br1, r2, r3, r4, r5, r6%5D%0A        assert set(ob.get_next_requests(10, overused_keys=%5B'www.example.com', 'example1.com'%5D,%0A                                        key_type='domain')) == set(%5Br4, r5%5D)%0A        assert set(self.logs) == set(%5B%22Overused keys: %5B'www.example.com', 'example1.com'%5D%22,%0A                                      %22Pending: 0%22%5D)%0A        self.logs = %5B%5D%0A%0A        assert ob.get_next_requests(10, overused_keys=%5B'www.example.com'%5D,%0A                                    key_type='domain') == %5Br6%5D%0A        assert set(self.logs) == set(%5B%22Overused keys: %5B'www.example.com'%5D%22,%0A                                     %22Pending: 4%22%5D)%0A        self.logs = %5B%5D%0A%0A        assert ob.get_next_requests(10, overused_keys=%5B'www.example.com'%5D,%0A                                    key_type='domain') == %5B%5D%0A        assert set(self.logs) == set(%5B%22Overused keys: %5B'www.example.com'%5D%22,%0A                                      %22Pending: 3%22%5D)%0A        self.logs = %5B%5D%0A%0A        #the max_next_requests is 3 here to cover the %22len(requests) == max_next_requests%22 case.%0A        assert set(ob.get_next_requests(3, overused_keys=%5B'example.com'%5D,%0A                                        key_type='domain')) == set(%5Br1, r2, r3%5D)%0A        assert set(self.logs) == set(%5B%22Overused keys: %5B'example.com'%5D%22,%0A                                      %22Pending: 3%22%5D)%0A        self.logs = %5B%5D%0A%0A        assert ob.get_next_requests(10, overused_keys=%5B%5D, key_type='domain') == %5B%5D%0A        assert set(self.logs) == set(%5B%22Overused keys: %5B%5D%22, %22Pending: 0%22%5D)%0A

test core.OverusedBuffer