import argparse
import codecs
import json
from getpass import getpass

from github3 import (
    GitHub,
    iter_user_repos,
    login
)

import gh_settings as settings


def dump_users(users):
    """Take a list of users as json and dump them to the json file for users as
    defined in settings."""

    fp = codecs.open(settings.USERS_DATASET, 'a', 'utf-8')
    json.dump(users, fp)
    fp.close()


def dump_repos(repos):
    """Take a list of repositories as json and dump them to the json file for
    repositories as defined in settings."""

    fp = codecs.open(settings.REPOS_DATASET, 'a', 'utf-8')
    json.dump(repos, fp)
    fp.close()


def main():
    user = input('GitHub username: ')
    password = getpass('GitHub password: ')

    if user and password:
        gh = login(user, password)
    else:
        gh = GitHub()

    # fetch none random developers
    devs = [user.login for user in
            gh.organization("DevMine").iter_public_members()]

    # fetch random developers
    for repo in gh.iter_all_repos(number=3000, since=3452093):
        if repo.private:
            continue
        devs.append(repo.owner)

    users = []
    repos = []

    # dump developers and their repositories
    for dev in devs:
        u = gh.user(dev)
        users.append(u.to_json())
        for repo in gh.iter_user_repos(u.login):
            repos.append(repo.to_json())

    dump_users(users)
    dump_repos(repos)


if __name__ == "__main__":
    main()

@@ -1,20 +1,4 @@
-import argparse%0A
 impo
@@ -19,16 +19,28 @@
 rt json%0A
+import time%0A
 from get
@@ -98,31 +98,27 @@
 ub,%0A    
+G
 it
-er_user_repos
+HubError
 ,%0A    lo
@@ -124,16 +124,17 @@
 ogin%0A)%0A%0A
+%0A
 import g
@@ -837,72 +837,1014 @@
   # 
-fetch none random developers%0A    devs = %5Buser.log
+It is defined inside the main because it makes use of the current gh%0A    def call_or_wait(function, *args, **kargs):%0A        '''Given a function and some arguments, tries to execute the function%0A        or waits until the ratelimit is reset.'''%0A%0A        success = False%0A        while not success:%0A            try:%0A                result = function(*args, **kargs)%0A                success = True%0A            except GitHubError as e:%0A                if e.code == 403:%0A                    # rate limit exceeded%0A                    reset = gh.rate_limit()%5B'rate'%5D%5B'reset'%5D%0A                    wait_time = int(reset - time.time()) + 1%0A%0A                    print(%22Not enough API calls. Wait
 in
+g
  for
- user in%0A
+%22,%0A                          int(wait_time / 60), %22minutes and%22,%0A                          wait_time %25 60, %22seconds%22)%0A%0A                    time.sleep(wait_time)%0A                else:%0A                    raise e%0A%0A        return result%0A%0A    # fetch none random developers%0A    devmine = call_or_wait(lambda:%0A          
     
@@ -1851,16 +1851,21 @@
         
+     
 gh.organ
@@ -1904,16 +1904,171 @@
 embers()
+)%0A    # the lambda is used to make the call lazy, as there are multiple API%0A    # calls involved in the statement%0A%0A    devs = %5Bdev.login for dev in devmine
 %5D%0A%0A    #
@@ -2100,32 +2100,45 @@
     for repo in 
+call_or_wait(
 gh.iter_all_repo
@@ -2138,17 +2138,18 @@
 ll_repos
-(
+, 
 number=3
@@ -2248,16 +2248,60 @@
 owner)%0A%0A
+    print(len(devs), %22developers fetched%22)%0A%0A
     user
@@ -2405,16 +2405,30 @@
 u = 
+call_or_wait(
 gh.user
-(
+, 
 dev)
@@ -2474,58 +2474,106 @@
     
-for repo in gh.iter_user_repos(u.login):%0A 
+if len(users) %25 500 == 0:%0A            print(%22Repos feched for%22, len(users), %22developers%22)%0A
         
    r
@@ -2572,37 +2572,61 @@
     
-   
 repos.
-app
+ext
 end(
-repo.to_json(
+call_or_wait(gh.iter_user_repos, u.login
 ))%0A%0A

make gh_fetch.py wait for more API calls