@@ -0,0 +1,2297 @@
+#! /usr/bin/python3%0A%0A'''%0Aautomate_scrape.py - runs throught the online verision of the text book, 'Automate the boring stuff with python' and pulls out all of the projects and stores them in a file.%0A'''%0A%0Aimport requests, os, bs4, sys%0A%0A%0Adef page_download(web_page, chapter_num, no_project_count, no_chapter_projects):%0A    '''%0A    Downloads the web page. Keeps the varibales, chapter_num, no_project_count, no_chapter_projects to be then used in chapter_loop_and_write. %0A    '''%0A%0A    print('Downloading page %7B0%7D'.format(web_page))%0A    res = requests.get(web_page)%0A    res.raise_for_status()%0A              %0A    soup = bs4.BeautifulSoup(res.text)%0A%0A    chapter_loop_and_write(soup, chapter_num, no_project_count, no_chapter_projects)%0A%0A%0Adef chapter_loop_and_write(downloaded_page, chapter_num, no_project_count, no_chapter_projects):%0A    '''%0A    Takes a downloaded web page and pulls out the practice projects and writes them to a file. It then moves to the next chapter, gets that url. Counts the number of chapters where practice projects could not be found and stores those chapters in a list.%0A    '''%0A    %0A    soup = downloaded_page%0A    projects = soup.find_all('div', %7B'class': %22book%22, 'title' : 'Practice Projects'%7D)%0A%0A    if projects == %5B%5D:%0A        print('Could not find Projects.')%0A        no_project_count += 1%0A        no_chapter_projects.append('Chapter' + str(chapter_num))%0A%0A    else:%0A        with open('automateProjects.txt', 'a') as f:%0A            for el in projects:%0A                f.write(el.get_text())%0A                print('Writing text to file')%0A        f.close()%0A%0A    chapter_num += 1%0A%0A    if chapter_num == 19:%0A        print('%5Cn%7B0%7D chapters where Practice Projects could not be found'.format(no_project_count))%0A        print('Here is the list of those chapters:%5Cn%7B0%7D'.format(no_chapter_projects))%0A        print('='*20 + 'Done' + '='*20 +'%5Cn')%0A        return%0A%0A    next_link = soup.find('a', href='/chapter' + str(chapter_num))%0A    web_page = 'http://automatetheboringstuff.com' + next_link.get('href') +'/'%0A%0A    page_download(web_page, chapter_num, no_project_count, no_chapter_projects)%0A%0A%0Adef main():%0A    '''%0A    main%0A    '''%0A%0A    web_page = 'https://automatetheboringstuff.com/chapter1/'%0A    %0A    page_download(web_page, 1, 0, %5B%5D)%0A%0A%0Aif __name__ == '__main__':%0A    sys.exit(main())%0A

Create automate_scrape.py