import json
import logging
import random
import urllib

from fake_useragent import UserAgent

from twitterscraper.tweet import Tweet


ua = UserAgent()
HEADERS_LIST = [ua.chrome, ua.google, ua['google chrome'], ua.firefox, ua.ff]

INIT_URL = "https://twitter.com/search?f=tweets&vertical=default&q={q}"
RELOAD_URL = "https://twitter.com/i/search/timeline?f=tweets&vertical=" \
             "default&include_available_features=1&include_entities=1&" \
             "reset_error_state=false&src=typd&max_position={pos}&q={q}"


def query_single_page(url, html_response=True):
    """
    Returns tweets from the given URL.

    :param url: The URL to get the tweets from
    :param html_response: False, if the HTML is embedded in a JSON
    :return: The list of tweets, the pos argument for getting the next page.
    """
    headers = {'User-Agent': random.choice(HEADERS_LIST)}
    req = urllib.request.Request(url, headers=headers)
    try:
        response = urllib.request.urlopen(req).read().decode()
        if html_response:
            html = response
        else:
            json_resp = json.loads(response)
            html = json_resp['items_html']

        tweets = list(Tweet.from_html(html))
        if not tweets:
            return [], None

        if not html_response:
            return tweets, json_resp['min_position']

        return tweets, "TWEET-{}-{}".format(tweets[-1].id, tweets[0].id)
    except urllib.request.HTTPError as e:
        logging.exception('HTTPError {} while requesting "{}"'.format(
            e.code, url))
    except urllib.request.URLError as e:
        logging.exception('URLError {} while requesting "{}"'.format(
            e.reason, url))

    return [], None


def query_tweets(query, limit=None):
    """
    Queries twitter for all the tweets you want!

    :param query: Any advanced query you want to do! Compile it at
                  https://twitter.com/search-advanced and just copy the query!
    :param limit: Scraping will be stopped when at least ``limit`` number of
                  items are fetched.
    :return:      A list of twitterscraper.Tweet objects. You will get at least
                  ``limit`` number of items.
    """
    query = query.replace(' ', '%20').replace("#", "%23")
    pos = None
    tweets = []
    while True:
        new_tweets, pos = query_single_page(
            INIT_URL.format(q=query) if pos is None
            else RELOAD_URL.format(q=query, pos=pos),
            pos is None
        )
        if len(new_tweets) == 0:
            return tweets

        tweets += new_tweets
        logging.info("Got {} tweets ({} new).".format(len(tweets),
                                                      len(new_tweets)))

        if limit is not None and len(tweets) >= limit:
            return tweets

@@ -1806,16 +1806,155 @@
  want!%0A%0A
+    Note that this function catches the KeyboardInterrupt so it can return%0A    tweets on incomplete queries if the user decides to abort.%0A%0A
     :par
@@ -2427,16 +2427,29 @@
 ts = %5B%5D%0A
+    try:%0A    
     whil
@@ -2464,16 +2464,20 @@
         
+  
+  
 new_twee
@@ -2505,16 +2505,20 @@
 e_page(%0A
+    
         
@@ -2569,24 +2569,28 @@
             
+ 
+   
 else RELOAD_
@@ -2615,24 +2615,28 @@
 , pos=pos),%0A
+    
             
@@ -2655,18 +2655,26 @@
         
-)%0A
+    )%0A    
         
@@ -2702,32 +2702,36 @@
  0:%0A            
+    
 return tweets%0A%0A 
@@ -2733,24 +2733,28 @@
 ts%0A%0A        
+  
+  
 tweets += ne
@@ -2762,16 +2762,20 @@
 _tweets%0A
+    
         
@@ -2887,16 +2887,20 @@
         
+    
 len(new_
@@ -2906,24 +2906,28 @@
 _tweets)))%0A%0A
+    
         if l
@@ -2969,24 +2969,193 @@
  limit:%0A    
+            return tweets%0A    except KeyboardInterrupt:%0A        logging.info(%22Program interrupted by user. Returning tweets gathered %22%0A                     %22so far...%22)%0A
         retu

Return tweets on program abortion