@@ -0,0 +1,3755 @@
+from scrapy.http import HtmlResponse%0Afrom logging import getLogger%0Aimport asyncio%0Aimport pyppeteer%0A%0A%0Aclass PyppeteerMiddleware():%0A    def __init__(self, **args):%0A        self.logger = getLogger(__name__)%0A        self.loop = asyncio.get_event_loop()%0A        self.browser = self.loop.run_until_complete(%0A            pyppeteer.launch(headless=True, args=%5B'--no-sandbox'%5D))%0A        self.args = args%0A    %0A    def render(self, url, retries=8, script=None, wait=0.2, scrolldown=False, sleep=0,%0A               timeout=8.0, keep_page=True, with_result=False):%0A        %22%22%22%0A        render page with pyppeteer%0A        :param url: page url%0A        :param retries: max retry times%0A        :param script: js script to evaluate%0A        :param wait: number of seconds to wait before loading the page, preventing timeouts%0A        :param scrolldown: how many times to page down%0A        :param sleep: how many long to sleep after initial render%0A        :param timeout: the longest wait time, otherwise raise timeout error%0A        :param keep_page: keep page not to be closed, browser object needed%0A        :param browser: pyppetter browser object%0A        :param with_result: return with js evaluation result%0A        :return: content, %5Bresult%5D%0A        %22%22%22%0A        # define async render%0A        async def async_render(url, script, scrolldown, sleep, wait, timeout, keep_page):%0A            try:%0A                # basic render%0A                page = await self.browser.newPage()%0A                await asyncio.sleep(wait)%0A                await page.goto(url, options=%7B'timeout': int(timeout * 1000)%7D)%0A                %0A                result = None%0A                # evaluate with script%0A                if script:%0A                    result = await page.evaluate(script)%0A                %0A                # scroll down for %7Bscrolldown%7D times%0A                if scrolldown:%0A                    for _ in range(scrolldown):%0A                        await page._keyboard.down('PageDown')%0A                        await asyncio.sleep(sleep)%0A                else:%0A                    await asyncio.sleep(sleep)%0A                if scrolldown:%0A                    await page._keyboard.up('PageDown')%0A                %0A                # get html of page%0A                content = await page.content()%0A                %0A                # if keep page, do not close it%0A                if not keep_page:%0A                    await page.close()%0A                %0A                return content, result%0A            except TimeoutError:%0A                return None%0A        %0A        content, result = None, None%0A        # retry for %7Bretries%7D times%0A        for i in range(retries):%0A            if not content:%0A                content, result = self.loop.run_until_complete(%0A                    async_render(url=url, script=script, sleep=sleep, wait=wait,%0A                                 scrolldown=scrolldown, timeout=timeout, keep_page=keep_page))%0A            else:%0A                break%0A        %0A        # if need to return js evaluation result%0A        if with_result:%0A            return content, result%0A        return content%0A    %0A    def process_request(self, request, spider):%0A        %22%22%22%0A        :param request: request object%0A        :param spider: spider object%0A        :return: HtmlResponse%0A        %22%22%22%0A        self.logger.debug('Render %25s', request.url)%0A        try:%0A            html = self.render(request.url)%0A            return HtmlResponse(url=request.url, body=html, request=request, encoding='utf-8',%0A                                status=200)%0A        except pyppeteer.errors.TimeoutError:%0A            return HtmlResponse(url=request.url, status=500, request=request)%0A    %0A    @classmethod%0A    def from_crawler(cls, crawler):%0A        return cls(**crawler.settings.get('PYPPETEER_ARGS'))%0A

add pyppeteer