@@ -0,0 +1,1346 @@
+# Usage: scrapy edxcourses.py %7C ... %7C maildir-putt%0A# -*- encoding: utf-8 -*-%0A%0Aimport json%0Aimport urlparse%0Aimport time%0Aimport scrapy%0A%0Aclass EdxCourses(scrapy.Spider):%0A    name = %22edx-courses%22%0A    start_urls = %5B%22https://www.edx.org/search/api/all%22%5D%0A%0A    def parse(self, response):%0A        for course in json.loads(response.body):%0A            if not u%22English%22 in course%5B%22languages%22%5D:%0A                continue%0A            if u%22profed%22 in course%5B%22types%22%5D:%0A                continue%0A            %0A            yield scrapy.Request(course%5B%22url%22%5D,%0A                    callback=lambda r:self.parse_course(r, course))%0A    %0A    def parse_course(self, response, summary):%0A        def do_parse(response):%0A            body = u%22%3Cp%3E%25s %E2%80%94 %25s (%25s)%3C/p%3E%22 %25 (summary%5B%22code%22%5D, summary%5B%22l%22%5D, summary%5B%22start%22%5D)%0A            body += u'%3Cp%3E%3Cimg src=%22%25s%22%3E%3C/p%3E' %25 (urlparse.urljoin(self.start_urls%5B0%5D, summary%5B%22image%22%5D%5B%22src%22%5D),)%0A            body += json.loads(response.body)%5B%22description%22%5D%0A%0A            print json.dumps(%7B%0A                %22url%22: summary%5B%22url%22%5D,%0A                %22title%22: u%22%25s %25s%22 %25 (u%22 %22.join(%22%5B%25s%5D%22%25_ for _ in summary%5B%22schools%22%5D), summary%5B%22l%22%5D),%0A                %22body%22: body%0A            %7D)%0A%0A        yield scrapy.Request(%22https://www.edx.org/api/catalog/v2/courses/%25s%22 %25 response.css('main ::attr(data-course-id)').extract()%5B0%5D,%0A                callback=do_parse)%0A

add scrapper edxcourses