import argparse
import contextlib
import cStringIO
import errno
import itertools
import json
import os
import re
import shutil
import sys
import zipfile

import bs4
import requests

BASE_URL = 'http://www.mathworks.com/matlabcentral/fileexchange'
TAG_REGEX = re.compile(r'([\w\s]+)(?:\(\d+\))?')


def get_soup(*args, **kwargs):
    return bs4.BeautifulSoup(requests.get(*args, **kwargs).text)


# From http://stackoverflow.com/a/600612/281108
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise


def extractall(archive, path):
    "A version of ZipFile.extractall that doesn't choke on utf-8 filenames."
    for filename in archive.namelist():
        dest_filename = os.path.join(path, filename.decode('cp437'))
        if dest_filename.endswith('/'):
            mkdir_p(dest_filename)
            continue
        mkdir_p(os.path.dirname(dest_filename))
        with archive.open(filename) as src, open(dest_filename, 'w') as dest:
            shutil.copyfileobj(src, dest)


class Project(object):
    def __init__(self, url_or_name):
        if url_or_name.startswith(BASE_URL):
            self.url = url_or_name
        else:
            self.url = '%s/%s' % (BASE_URL, url_or_name)

    def __repr__(self):
        return '<Project: %s>' % self.name

    @property
    def name(self):
        return self.url.split('/')[-1]

    # The way downloads work on MatlabCentral is that the download link
    # redirects you to the url of the actual file you're downloading.
    # Instead of following the redirect automatically, we manually get
    # the value of the Location: header; this lets us get at the name of
    # the downloaded file (and so we can tell whether it's a zip, etc.)
    def download(self, path='.', extract_archives=True):
        download_url = '%s?download=true' % self.url
        response = requests.head(download_url, allow_redirects=False)
        real_download_url = response.headers['Location']
        download_filename = real_download_url.split('/')[-1]
        response = requests.get(real_download_url)
        if extract_archives and download_filename.endswith('.zip'):
            with zipfile.ZipFile(cStringIO.StringIO(response.content)) as f:
                extractall(f, path)
        else:
            with open(os.path.join(path, download_filename), 'w') as f:
                f.write(response.text.encode('utf-8'))

    def get_metadata(self):
        metadata = {}
        soup = get_soup(self.url)

        tags_div = soup.find('div', id='all_tags')
        raw_tags = tags_div.get_text().strip().split('\n\n')[0].split(', ')
        tags = []
        for tag in raw_tags:
            match = TAG_REGEX.match(tag)
            if match:
                tags.append(match.group(1))
        metadata['tags'] = tags

        details = soup.find('div', id='details')

        metadata['title'] = details.h1.text
        metadata['summary'] = details.find('p', id='summary').text

        author_link = details.find('p', id='author').a
        metadata['author'] = author_link.text
        metadata['author_url'] = author_link['href']

        metadata['date_submitted'] = details.find(
            'span', id='submissiondate').text.strip()
        date_span = details.find('span', id='date_updated')
        if date_span is not None:
            metadata['date_updated'] = date_span.text[len('(Updated '):-1]
        return metadata

    def get_json(self):
        return dict(name=self.name, url=self.url, **self.get_metadata())


def fileindex_projects(num_projects, sort):
    def helper():
        for page in itertools.count(1):
            params = dict(page=page, term='type:Function', sort=sort)
            soup = get_soup(BASE_URL, params=params)
            for title in soup.find_all('p', attrs={'class': 'file_title'}):
                yield Project(title.a['href'])
    return itertools.islice(helper(), num_projects)


def parse_args():
    parser = argparse.ArgumentParser(
        description='download files from MatlabCentral')
    parser.add_argument(
        '--num_projects', default=10, type=int,
        help='Number of projects to fetch.')
    parser.add_argument(
        '--extract_archives', type=bool, default=True,
        help='If true, automatically extract archives.')
    SORT_CHOICES = ('downloads_desc', 'downloads_asc',
                    'date_desc_updated', 'date_asc_updated',
                    'date_desc_submitted', 'date_asc_submitted',
                    'comments_desc', 'comments_asc',
                    'ratings_desc', 'ratings_asc')
    parser.add_argument(
        '--sort', choices=SORT_CHOICES, default='downloads_desc',
        help='Sorting criteria.')
    parser.add_argument(
        '--to', default='',
        help='Directory to download projects to (default current).')
    return parser.parse_args()


@contextlib.contextmanager
def status(message):
    print '%s...' % message,
    sys.stdout.flush()
    yield
    print 'done.'


def main():
    args = parse_args()
    projects = []
    for project in fileindex_projects(args.num_projects, args.sort):
        download_path = os.path.join(args.to, project.name)
        os.makedirs(download_path)
        with status('Downloading %s' % project.name):
            project.download(download_path, args.extract_archives)
        projects.append(project.get_json())
    with open(os.path.join(args.to, 'manifest.json'), 'w') as f:
        json.dump({'projects': projects}, f, indent=2, sort_keys=True)

if __name__ == '__main__':
    main()

@@ -1468,366 +1468,8 @@
 1%5D%0A%0A
-    # The way downloads work on MatlabCentral is that the download link%0A    # redirects you to the url of the actual file you're downloading.%0A    # Instead of following the redirect automatically, we manually get%0A    # the value of the Location: header; this lets us get at the name of%0A    # the downloaded file (and so we can tell whether it's a zip, etc.)%0A
     
@@ -1602,20 +1602,19 @@
 equests.
-head
+get
 (downloa
@@ -1622,89 +1622,9 @@
 _url
-, allow_redirects=False)%0A        real_download_url = response.headers%5B'Location'%5D
+)
 %0A   
@@ -1650,28 +1650,23 @@
 ame = re
-al_download_
+sponse.
 url.spli
@@ -1680,59 +1680,8 @@
 -1%5D%0A
-        response = requests.get(real_download_url)%0A
     

Simplify downloading.