<jupyter_start><jupyter_text>Import WallStreetBets Reddit comment data Comments take hours to all load, it is about 2.5 million comments therefore the collected data has been prestored in a csv<jupyter_code># import reddit comment collection function 
from RedditScraper import *

# Create dataframe of reddit comments of the last 60 days on wallstreetbets
reddit_df = collect_subreddit_comments(after='60d', subreddit='wallstreetbets', columns=['body', 'score', 'created_utc'])

# Save dataframe as csv
reddit_df.to_csv('reddit_comments.csv')
reddit_df = pd.read_csv('reddit_comments.csv')<jupyter_output>/home/sedawarp/anaconda3/envs/tensorflow/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3166: DtypeWarning: Columns (0) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>PREPROCESSING<jupyter_code># import preprocessor class and preprocess comments and add datatime variables
from TextPreprocessor import Preprocessor 

reddit_df = Preprocessor().preprocess_reddit(reddit_df)
reddit_df.tail(20)<jupyter_output><empty_output><jupyter_text>Ticker Analyisis - What is currently popular?  Finding all comments mentioning selected stocks and counting freqquency<jupyter_code>from StockParser import WSBStockParser

stockScanner = WSBStockParser()

reddit_df_filtered = stockScanner.find_mentions(reddit_df)
reddit_df_filtered['Stock'].value_counts()<jupyter_output><empty_output><jupyter_text>Generate Sentiment  Vader<jupyter_code># Import self made vader class
from SentimentGenerator import Vader

# initalize vader object for sentiment generation
v_sentiment = Vader()

# assign new vader sentiment column and weighted sentiment 
reddit_df_filtered = v_sentiment.classify_list(reddit_df_filtered)
reddit_df_filtered.head()<jupyter_output><empty_output><jupyter_text>ROBERTA<jupyter_code># Imported self created Roberta class for sentiment generation
from SentimentGenerator import OpinionatedRoberta 

# Create sentiment generator object 
b_sentiment = OpinionatedRoberta()
# loop through all comments to generate scores

scores = []

for comment in reddit_df_filtered.body.values:
    
    try:
        sentiment_probs = list(b_sentiment.classify(comment))
        prediction = sentiment_probs.index(max(sentiment_probs))
        scores.append(prediction)
    
    except:
        # if edgecase occurs output neutral sentiment 
        print(comment)
        scores.append(1)

reddit_df_filtered['bert_sentiment'] = scores
reddit_df_filtered['bert_sentiment'].value_counts()
reddit_df_filtered.head()<jupyter_output><empty_output><jupyter_text>To run the RoBERTa model on the full dataset takes a few hours so the final data set is saved as a csv<jupyter_code>reddit_df_filtered = pd.read_csv('reddit_sentiment_bert.csv')<jupyter_output><empty_output><jupyter_text>Predicting next day direction with sentiment<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold, cross_validate, KFold
import yfinance as yf
import numpy as np
import datetime 
import pytz<jupyter_output><empty_output><jupyter_text>Importing stock data from yfinance<jupyter_code>import yfinance as yf
import numpy as np
import datetime 
import pytz

stock_list = ['gme', 'amc', 'wish', 'spy', 'clov','bb']

def get_closing_values(stocks = stock_list, start='2021-06-18', end='2021-08-16', interval='1h'):
    
    # Create empty dataframe to populate stock close into 
    price_df = pd.DataFrame()
    
    # Loop through stock list to collect closing prices 
    for stock in stocks:
        stock_connection = yf.Ticker(stock.upper())
        hist = stock_connection.history(start='2021-06-18', end='2021-08-16', interval='1h')
        close_price = hist['Close']
        price_df[stock] = close_price
    
    
    # Create timezone to UTC
    def change_timezone(time):
        local = pytz.timezone("America/New_York")
        naive = datetime.datetime.strptime(time, "%Y-%m-%d %H")
        local_dt = local.localize(naive, is_dst=None)
        utc_dt = local_dt.astimezone(pytz.utc)
        
        return utc_dt
    
    # Adjusting dates to UTC
    price_df['date'] = hist.index
    price_df['date'] = price_df.apply(lambda row: row['date'].strftime('%Y-%m-%d %H'), axis=1)
    price_df['date'] = price_df.apply(lambda row: change_timezone(row.date).strftime('%Y-%m-%d %H'), axis=1)
    price_df = price_df.reset_index(drop=True)
    
    # Melt dataframe 
    price_df = pd.melt(price_df, id_vars='date', var_name='Stock', value_name='close')
    
    return price_df


     
price_df = get_closing_values()
price_df.shape<jupyter_output><empty_output><jupyter_text>Create hourly price/sentiment data<jupyter_code>from scipy.stats import mode

# group sentiment by family 
hourly_sentiment = reddit_df_filtered.groupby(by=['Stock','date']).agg({'vader_sentiment':'mean', 'weighted_sentiment':'mean', 'Stock':'count','bert_sentiment':lambda x: mode(x)[0][0]}).rename(columns={'Stock':'count'}).reset_index()

# Merge hourly stock price data with hourly sentiment data for each stock
price_sentiment = pd.merge(hourly_sentiment, price_df, how='inner', on=['Stock','date'])

price_sentiment.head(10)<jupyter_output><empty_output><jupyter_text>Final Feature enginnering prior to Logistic Regression<jupyter_code>def verbal_score(x):
    
    if x >=0.01:
        return 'positive'
    
    elif x < 0.01 and x > -0.01: 
        return 'neutral'
    
    else:
        return 'negative'

def bert_score(x):
    
    if x == 2:
        return 'positive'
    
    elif x == 1:
        return 'neutral'
    
    else:
        return 'negative'

# Change scores into positive, negative, and neutral 
price_sentiment['vader_score'] = price_sentiment['vader_sentiment'].apply(lambda c: verbal_score(c))
price_sentiment['weighted_score'] = price_sentiment['weighted_sentiment'].apply(lambda c: verbal_score(c))
price_sentiment['bert_score'] = price_sentiment['bert_sentiment'].apply(lambda c: bert_score(c))
price_sentiment['vader_score'].value_counts()
price_sentiment['weighted_score'].value_counts()
price_sentiment['bert_score'].value_counts()<jupyter_output><empty_output><jupyter_text>Looping through all stocks and generating roc-auc results<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold, cross_validate, KFold
from sklearn.ensemble import RandomForestClassifier

model_results_df = pd.DataFrame()

for stock in stock_list:
    stock_df = price_sentiment.copy()
    stock_df = stock_df[stock_df['Stock'] == stock]
    
    # apply log-change 
    stock_df['log_change'] = np.log(stock_df.close) - np.log(stock_df.close.shift(1))
    
    # Create binary label for prediction 
    stock_df = stock_df.dropna()
    stock_df['direction'] = stock_df['log_change'].apply(lambda x: 1 if x >0 else 0)
    label_imbalance = stock_df.direction.value_counts()[0]/stock_df.direction.count()
    
    # Create t-1 price change variable 
    stock_df['t-1'] = stock_df.log_change.shift(1)
    
    for focus in ['weighted_score','vader_score','count','normal', 'bert_score']:
        
        if focus in ['weighted_score','vader_score','bert_score']:
    
            dummies = pd.get_dummies(stock_df[focus])
            stock_df = pd.concat([stock_df, dummies], axis=1)
            stock_df = stock_df.dropna()
            
            try:
                X = stock_df[['negative','neutral','positive','t-1']]
                y = stock_df['direction']
            
            except:
                X = stock_df[['negative','positive','t-1']]
                y = stock_df['direction']
        
        elif focus == 'normal':
            X = stock_df[['t-1']]
            y = stock_df['direction']
        
        else:
            X = stock_df[['count','t-1']]
            y = stock_df['direction']
            
                
        X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=100, shuffle=True)

        scaler = StandardScaler()
        X_train = scaler.fit_transform(X_train)
        X_test = scaler.transform(X_test)

        clf = LogisticRegression()
        clf.fit(X_train, y_train)

        y_hat = clf.predict(X_test) 
        accuracy = (y_hat==y_test).mean() 
        
        k_fold = KFold(n_splits=5, shuffle=True, random_state=0)
        output = cross_validate(clf, X, y, cv=k_fold, scoring = 'roc_auc', return_estimator =False)['test_score'].mean()
        model_results_df = model_results_df.append({'Stock': stock, 'Variable':focus, 'Positive Imbalance': np.round(label_imbalance, 2), 'ROC_AUC': np.round(output, 2), 'Accuracy':np.round(accuracy, 2)}, ignore_index=True)

model_results_df = model_results_df[['Stock','Positive Imbalance', 'Variable', 'ROC_AUC', 'Accuracy']]
    
model_results_df
model_results_df.to_csv('model_results.csv')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup