<jupyter_start><jupyter_text>import  user input<jupyter_code>from google.colab import drive
drive.mount('drive')
# drive = 'drive/My Drive/Spring 2021/Stringer/Data/'
drive = '/content/drive/MyDrive/Stringer/Data/'
# # Twitter key
# ACCESS_TOKEN = 
# ACCESS_TOKEN_SECRET = 
# CONSUMER_KEY = 
# CONSUMER_SECRET = 
# code to ingest user input and then turn it into list of search terms

# searchTerm = [x for x in input("Please enter a list of query terms\n").split(', ')]
# print("\nInput Values: ", searchTerm) # vaccine, covid, coronavirus, pfizer, moderna, astrazenica, johnson, jab
# enter in query requirements, number of tweets, user, etc. 
startDate = '2021-04-24'
endDate = '2021-04-25' 
# searchTerm = ['vaccine', 'covid']
searchTerm = ['vaccine', 'covid', 'coronavirus', 'pfizer', 'moderna', 'astrazenica', 'johnson', 'jab']

numTweets = 15000
# Please change to reflect user 
user = 'RHI' #PS #KE #RHI #FP 

def timeStamped(fname, fmt='%Y-%m-%d-%H-%M-%S-{fname}'):
        import datetime
        return datetime.datetime.now().strftime(fmt).format(fname=fname)

outputFile = timeStamped(f'{user}_{numTweets}') + '_' + startDate
print(outputFile)<jupyter_output>2021-05-01-23-52-42-RHI_15000_2021-04-24<jupyter_text>packages<jupyter_code>import pandas as pd 
import numpy as np
import re

!pip install ipython-autotime
%load_ext autotime

!pip install tweepy
import tweepy

import csv

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

# create API object
api = tweepy.API(auth)<jupyter_output>Collecting ipython-autotime
  Downloading https://files.pythonhosted.org/packages/b4/c9/b413a24f759641bc27ef98c144b590023c8038dfb8a3f09e713e9dff12c1/ipython_autotime-0.3.1-py2.py3-none-any.whl
Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from ipython-autotime) (5.5.0)
Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (5.0.5)
Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (56.0.0)
Requirement already satisfied: pexpect; sys_platform != "win32" in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (4.8.0)
Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (1.0.18)
Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (0.8.1)
Requiremen[...]<jupyter_text>option 1: search<jupyter_code># edit user input so that it works with twitter search
# https://developer.twitter.com/en/docs/twitter-api/v1/tweets/search/guides/standard-operators 
searchTermStr = str(searchTerm).replace(',', ' OR')
searchTermStr = re.sub("'|\[|\]", '', searchTermStr)
searchTermStr
# choose features and save to file
outputPath = str(outputFile + '_search' + '.csv')

tweets = tweepy.Cursor(api.search, q=searchTermStr, tweet_mode = 'extended', lang='en', since=startDate, until=endDate, wait_on_rate_limit=True).items(numTweets)
# tweets = tweepy.Cursor(api.search, q=searchTermStr, tweet_mode = 'extended', lang='en', wait_on_rate_limit=True).items(numTweets) #, since=startDate, until=endDate)
tweetLst = []
i = 0
for tweet in tweets:
  if i % 500 == 0:
    print(i, 'tweets so far')

  if hasattr(tweet, 'retweeted_status'):  # Check if Retweet
    try:
      tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.retweeted_status.extended_tweet["full_text"], tweet.favorite_count, tweet.retweet_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # print(tweet.retweeted_status.extended_tweet["full_text"], '\n', [i['text'] for i in tweet.entities['hashtags']])
    except:
      tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.retweeted_status.full_text, tweet.favorite_count, tweet.retweet_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.retweeted_status.text, tweet.favorite_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # print(tweet.retweeted_status.text, '\n', [i['text'] for i in tweet.entities['hashtags']])
  else:
    try:
      tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.extended_tweet["full_text"], tweet.favorite_count, tweet.retweet_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # print(tweet.extended_tweet["full_text"], '\n', [i['text'] for i in tweet.entities['hashtags']])
    except:
      tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.full_text, tweet.favorite_count, tweet.retweet_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # tweetLst.append([tweet.id_str, tweet.author.screen_name, tweet.created_at, tweet.text, tweet.favorite_count, tweet.user.description, tweet.user.verified, tweet.user.followers_count, 1, [i['text'] for i in tweet.entities['hashtags']]])
      # print(tweet.text, '\n', [i['text'] for i in tweet.entities['hashtags']])
  if hasattr(tweet, 'quoted_status'):   # Check if Quote tweet
    # if hasattr(tweet.quoted_status) #, 'extended_tweet'):
    # tweetLst.append([1, tweet.quoted_status.extended_tweet['full_text']]) #[status.quoted_status_id, tweet_mode='extended'])
  # else:
    tweetLst[i].extend([1, tweet.quoted_status.full_text])
  else:
    tweetLst[i].extend([0, 'None'])
  i +=1
print(len(tweetLst), tweetLst[:10])

with open(outputPath, "w", newline="") as f:
    writer = csv.writer(f)
    writer.writerow(['tweet_id', 'user', 'created_on', 'tweet_text', 'fav_count', 'retweet_count', 'user_description', 'user_verified', 'user_follower_count', 'is_retweet', 'hashtags', 'is_quote', 'quote_text'])
    writer.writerows(tweetLst)
print('Done')<jupyter_output>0 tweets so far
500 tweets so far
1000 tweets so far
1500 tweets so far
2000 tweets so far
2500 tweets so far
3000 tweets so far
3500 tweets so far
4000 tweets so far
4500 tweets so far
5000 tweets so far
5500 tweets so far
6000 tweets so far
6500 tweets so far
7000 tweets so far
7500 tweets so far
8000 tweets so far
8500 tweets so far
9000 tweets so far
9500 tweets so far
10000 tweets so far
10500 tweets so far
11000 tweets so far
11500 tweets so far
12000 tweets so far
12500 tweets so far
13000 tweets so far
13500 tweets so far
14000 tweets so far
14500 tweets so far
15000 [['1386107687465848835', 'NoohpotoFarooq', datetime.datetime(2021, 4, 24, 23, 59, 59), 'Emergency Meeting On Current Situation Of Covid-19 Outbreak. \nLocals Are Requested To Follow SOPs To Get Rid Of Deadly Virus.\nDate :- 25-04-2021\nTime :- 12:30 AM \n#COVID19outbreak\nHere is @lalajaffar45 \nDHO Hyderabad\nHats off to His Struggle and his teams work\nMore Power to them https://t.co/RMQemNVg7F', 0, 26, 'politica[...]<jupyter_text>option 2: stream listener<jupyter_code># https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/overview
# choose features and save data
outputPath = str(drive + '/streamListenerRaw/' + outputFile + '_streamlistener' + '.csv')

class CustomStreamListener(tweepy.StreamListener):
    def __init__(self, api=None):
      super(CustomStreamListener, self).__init__()  
      self.num_tweets = 0

    def on_status(self, status):
        #record = {'Text': status.text, 'Created At': status.created_at}
        #print('record:', record)
        self.num_tweets += 1
        
        #this is where you dictate the number of tweets for streamlistener
        if self.num_tweets < 10000:
          
       
          if hasattr(status, "retweeted_status"):  # Check if Retweet
            try:
              if hasattr(status, 'quoted_status'):   # Check if Quote tweet
                try:
                  print(status.retweeted_status.extended_tweet["full_text"], '\n', [i['text'] for i in status.entities['hashtags']], status.quoted_status.full_text)

                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.retweeted_status.extended_tweet["full_text"], status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 1, [i['text'] for i in status.entities['hashtags']], 1, status.quoted_status.full_text])

                except:
                  print(status.retweeted_status.extended_tweet["full_text"], '\n', [i['text'] for i in status.entities['hashtags']], 'None')

                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.retweeted_status.extended_tweet["full_text"], status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 1, [i['text'] for i in status.entities['hashtags']], 0, 'None'])
              
            except AttributeError:
              if hasattr(status, 'quoted_status'):   # Check if Quote tweet
                try:
                  print(status.retweeted_status.text, '\n', [i['text'] for i in status.entities['hashtags']], status.quoted_status.full_text)
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.retweeted_status.text, status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 1, [i['text'] for i in status.entities['hashtags']], 1, status.quoted_status.full_text])

                except:
                  print(status.retweeted_status.text, '\n', [i['text'] for i in status.entities['hashtags']], 'None')
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.retweeted_status.text, status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 1, [i['text'] for i in status.entities['hashtags']], 0, 'None'])

          else:
            try:
              if hasattr(status, 'quoted_status'):   # Check if Quote tweet
                try:
                  print(status.extended_tweet["full_text"], '\n', [i['text'] for i in status.entities['hashtags']], status.quoted_status.full_text)
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.extended_tweet["full_text"], status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 0, [i['text'] for i in status.entities['hashtags']], 1, status.quoted_status.full_text])
                
                except:
                  print(status.extended_tweet["full_text"], '\n', [i['text'] for i in status.entities['hashtags']], 'None')             
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.extended_tweet["full_text"], status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 0, [i['text'] for i in status.entities['hashtags']], 0, 'None'])
            
            except AttributeError:
              if hasattr(status, 'quoted_status'):   # Check if Quote tweet
                try:
                  print(status.text, '\n', [i['text'] for i in status.entities['hashtags']], status.quoted_status.full_text)
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.text, status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 0, [i['text'] for i in status.entities['hashtags']], 1, status.quoted_status.full_text])
                
                except:
                  print(status.text, '\n', [i['text'] for i in status.entities['hashtags']], 'None')
                  
                  # Writing status data
                  with open(outputPath, 'a') as f:
                    writer = csv.writer(f) #, quotechar = "'")
                    writer.writerow([status.id_str, status.author.screen_name, status.created_at, status.text, status.favorite_count, status.user.description, status.user.verified, status.user.followers_count, 0, [i['text'] for i in status.entities['hashtags']], 0, 'None'])
          return True

        else:
            return False

    def on_error(self, status_code):
        print >> sys.stderr, 'Error:', status_code
        return True 

    def on_timeout(self):
        print >> sys.stderr, 'Timeout.'
        return True 

# Writing csv titles
with open(outputPath, 'w') as f:
    writer = csv.writer(f)
    # writer.writerow(['tweet_id', 'user', 'created_on', 'tweet_text', 'fav_count', 'user_description', 'user_verified', 'user_follower_count', 'is_retweet', 'hashtags'])
    writer.writerow(['tweet_id', 'user', 'created_on', 'tweet_text', 'fav_count', 'user_description', 'user_verified', 'user_follower_count', 'is_retweet', 'hashtags', 'is_quote', 'quote_text'])

streamingAPI = tweepy.streaming.Stream(auth, CustomStreamListener())
streamingAPI.filter(track=searchTerm, languages=['en'], is_async=True)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup