#!/usr/bin/env python
#

import sys, os, argparse, logging, fnmatch, posixpath, socket
import github

if sys.version_info < (3, 0):
    # python 2
    import urlparse
    from urllib import urlretrieve
else:
    # python 3
    import urllib.parse as urlparse
    from urllib.request import urlretrieve

def main(args, loglevel):
    logging.basicConfig(format="%(levelname)s: %(message)s", level=loglevel)

    socket.setdefaulttimeout(args.timeout)

    if args.username and args.password:
      g = github.Github(args.username, args.password)
    else:
      g = github.Github()

    if args.repo_file:
        with open(args.repo_file, 'r') as f:
            download_files(args, g, f)
    else:
        download_files(args, g, repo_gen(args, g))


def repo_gen(args, g):
    last_page = args.last_page
    git_repo_gen = g.get_repos(since=args.last_page)
    for repo in git_repo_gen:
        current_page = git_repo_gen.__nextUrl
        if current_page != last_page:
            print("Finished processing: {}".format(current_page))
            last_page = current_page
        yield repo


def download_files(args, g, repo_gen):
    file_counter = 0
    for line in repo_gen:
        logging.info('Fetching repository: %s' % line)
        try:
            repo_str = line.rstrip().split('github.com/')[-1]
            repo = g.get_repo(repo_str)
            tree = repo.get_git_tree('master', recursive=True)
            files_to_download = []
            for file in tree.tree:
                if fnmatch.fnmatch(file.path, args.wildcard):
                    files_to_download.append('https://github.com/%s/raw/master/%s' % (repo_str, file.path))
            for file in files_to_download:
                logging.info('Downloading %s' % file)
                file_counter += 1
                filename = posixpath.basename(urlparse.urlsplit(file).path)
                output_path = os.path.join(args.output_dir, filename)
                if os.path.exists(output_path):
                    output_path += "-" + str(file_counter)
                try:
                    urlretrieve(file, output_path)
                except Exception:
                    logging.exception('Error downloading %s.' % file)
        except Exception:
             logging.exception('Error fetching repository %s.' % line)

    args.yara_meta = os.path.join(args.output_dir, args.yara_meta)
    with open(args.yara_meta, 'w') as f:
        for i in os.listdir(args.output_dir):
            try:
                f.write("include \"" + i + "\"\n")
            except Exception:
                logging.exception('Couldn\'t write to %s.' % args.yara_meta)



if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = "Github file downloader")
    parser.add_argument("-u",
                        "--username",
                        default = None,
                        help = "Username used to authenticate with github for increased rate limit")

    parser.add_argument("-p",
                        "--password",
                        default = None,
                        help = "Password or token used to authenticate with github")

    parser.add_argument("-r",
                        "--repo_file",
                        help = "Path for the input file which contains a url of a Github repository for each separate line")

    parser.add_argument("-l",
                        "--last_page",
                        default = github.GithubObject.NotSet,
                        help = "When not using a repo_file, this will be used as starting position for github repo crawl")

    parser.add_argument("-w",
                        "--wildcard",
                        help = "Unix shell-style wildcard to match files to download (for example: *.txt)")

    parser.add_argument("-o",
                        "--output_dir",
                        default = "",
                        help = "Directory to store all downloaded files")

    parser.add_argument("-y",
                        "--yara-meta",
                        default = "rules.yara",
                        help = "Yara meta rule filename to create")

    parser.add_argument("-t",
                        "--timeout",
                        default = 30,
                        help = "Socket timeout (seconds)")

    parser.add_argument("-v",
                        "--verbose",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()

    # Setup logging
    if args.verbose:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    main(args, loglevel)

@@ -610,79 +610,104 @@
     
-with open(args.repo_file, 'r') as f:%0A            download_files
+repo_gen = file_repo_gen(args.repo_file, g)%0A    else:%0A        repo_gen = global_repo_gen
 (args, g
 , f)
@@ -706,27 +706,10 @@
 s, g
-, f
 )%0A
-    else:%0A    
     
@@ -744,25 +744,20 @@
 _gen
-(args, g))%0A
+)
 %0A%0Adef 
+file_
 repo
@@ -757,28 +757,33 @@
 le_repo_gen(
-args
+repo_file
 , g):%0A    la
@@ -784,304 +784,278 @@
     
-last_page = args.last_page%0A    git_repo_gen = g.get_repos(since=args.last_page)%0A    for repo in git_repo_gen:%0A        current_page = git_repo_gen.__nextUrl%0A        if current_page != last_page:%0A            print(%22Finished processing: %7B%7D%22.format(current_page))%0A            last_page = current_page
+with open(repo_file, 'r') as f:%0A        for line in f:%0A            repo_str = line.rstrip().split('github.com/')%5B-1%5D%0A            yield g.get_repo(repo_str)%0A%0A%0Adef global_repo_gen(args, g):%0A    git_repo_gen = g.get_repos(since=args.last_repo)%0A    for repo in git_repo_gen:
 %0A   
@@ -1136,28 +1136,28 @@
 = 0%0A    for 
-line
+repo
  in repo_gen
@@ -1211,128 +1211,36 @@
 ' %25 
-line)%0A        try:%0A            repo_str = line.rstrip().split('github.com/')%5B-1%5D%0A            repo = g.get_repo(repo_str)
+repo.full_name)%0A        try:
 %0A   
@@ -1521,20 +1521,26 @@
  %25 (repo
-_str
+.full_name
 , file.p
@@ -2196,19 +2196,29 @@
  %25s.' %25 
-lin
+repo.full_nam
 e)%0A%0A    
@@ -3314,13 +3314,47 @@
 ast_
-page%22
+repo%22,%0A                        type=int
 ,%0A  

Fix errors when using repo_file, layout, drop printing last page requests