import requests
from bs4 import BeautifulSoup


def download_url(url):
    r = requests.get(url)
    if r.status_code != 200:
        raise Exception('Non-OK status code: {}'.format(r.status_code))
    return r.text


def parse_text(html):
    bs = BeautifulSoup(html, 'html.parser')
    return bs.select('div.usertext-body')[1].text


@@ -39,16 +39,72 @@
 fulSoup%0A
+import os.path%0Afrom base64 import b16encode%0Aimport time%0A
 %0A%0Adef do
@@ -110,16 +110,23 @@
 ownload_
+reddit_
 url(url)
@@ -123,24 +123,153 @@
 t_url(url):%0A
+    assert url.startswith('https://www.reddit.com/r/learnprogramming')%0A    headers = %7B'User-Agent':'JustSearch bot version 0.1'%7D%0A
     r = requ
@@ -280,16 +280,33 @@
 .get(url
+, headers=headers
 )%0A    if
@@ -430,19 +430,26 @@
 f parse_
-tex
+reddit_pos
 t(html):
@@ -540,12 +540,1149 @@
 ')%5B1%5D.text%0A%0A
+%0Aclass Crawler(object):%0A    def __init__(self, start_url, storage_dir):%0A        self.start_url = start_url%0A        self.storage_dir = storage_dir%0A%0A    @staticmethod%0A    def _make_absolute_url(url):%0A        return 'https://www.reddit.com' + url%0A%0A    def crawl(self):%0A        current_page_url = self.start_url%0A        while True:%0A            current_page = download_reddit_url(current_page_url)%0A            bs = BeautifulSoup(current_page)%0A            all_posts_links = bs.findAll('a', attrs=%7B'class': 'title'%7D)%0A            post_links = %5BCrawler._make_absolute_url(link%5B'href'%5D) for link in all_posts_links%5D%0A            for post_link in post_links:%0A                html = download_reddit_url(post_link)%0A                stored_text_file_name = os.path.join(self.storage_dir, b16encode(post_link))%0A                stored_text_file = open(stored_text_file_name, 'w')%0A                stored_text_file.write(html.encode('utf8'))%0A                stored_text_file.close()%0A            next_page_url = bs.find('a', attrs=%7B'rel':'next'%7D)%5B'href'%5D%0A            current_page_url = next_page_url%0A            time.sleep(2)%0A            print  next_page_url

Create files with posts and can go to the next page.