from ..darkobject import DarkObject
from bs4 import BeautifulSoup
import logging
import time


from urllib.request import urlopen


class Scrubber(DarkObject):
    def __init__(self):
        super(Scrubber, self).__init__()

    def scrub(self):
        """
        Get item metadata.
        """
        return {}

    # noinspection PyBroadException
    @staticmethod
    def soup(url):
        """
        Open URL and create tag soup.

        :param url: website string
        :type url: str
        """
        handle = ''
        max_tries = 10
        for i in range(max_tries):
            # noinspection PyPep8
            try:
                handle = urlopen(url)
                handle = handle.read()
                break
            except:
                logging.exception('urlopen failed (attempt %d)', i + 1)
                if i == max_tries - 1:
                    logging.error('the maximum urlopen attempts have been reached')
                    raise
                time.sleep(1)

        s = BeautifulSoup(handle, features='html5lib')
        return s


if __name__ == '__main__':
    pass

@@ -59,16 +59,32 @@
 fulSoup%0A
+import requests%0A
 import l
@@ -108,45 +108,8 @@
 e%0A%0A%0A
-from urllib.request import urlopen%0A%0A%0A
 clas
@@ -359,16 +359,50 @@
 soup(url
+, absolute: bool = False, **kwargs
 ):%0A     
@@ -504,16 +504,119 @@
 url: str
+%0A%0A        :param absolute: perform double get request to find absolute url%0A        :type absolute: bool
 %0A       
@@ -769,29 +769,68 @@
     
-handle = urlopen
+if absolute:%0A                    url = requests.get
 (url)
+.url
 %0A   
@@ -855,21 +855,45 @@
 e = 
-handle.read()
+requests.get(url, params=kwargs).text
 %0A   
@@ -929,16 +929,26 @@
   except
+ Exception
 :%0A      

Add support for absolute URL in request