<jupyter_start><jupyter_text>Creat Vote Matrix and Bill representation<jupyter_code>model = gensim.models.KeyedVectors.load_word2vec_format('/home/sonic/.keras/datasets/GoogleNews-vectors-negative300.bin',
                                                        binary=True)

  
max_words = 20000
MAX_SEQUENCE_LENGTH = 10000
def process_doc(X):
    tokenizer = Tokenizer(num_words=max_words,lower=True, split=' ', 
                          filters='"#%&()*+-/<=>@[\\]^_`{|}~\t\n',
                          char_level=False, oov_token=u'<UNK>')

    tokenizer.fit_on_texts(X)

    X_seq = tokenizer.texts_to_sequences(X)

    count_vect = CountVectorizer()
    X_train_counts = count_vect.fit_transform(X)

    tf_transformer = TfidfTransformer().fit(X_train_counts)
    X_train_tf = tf_transformer.transform(X_train_counts)

    x_emb =[]
    for doc in X: #look up each doc in model
        x_emb.append(document_vector(model, doc))


    word_index = tokenizer.word_index
    print('Found %s unique tokens.' % len(word_index))

#     X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH,  padding="post", truncating="post")
    return np.array(X_seq), word_index, np.array(x_emb), X_train_tf, X_train_counts

def document_vector(word2vec_model, doc):
    # remove out-of-vocabulary words
    doc = [word for word in doc if word in word2vec_model.vocab]
    return np.mean(word2vec_model[doc], axis=0)


def has_vector_representation(word2vec_model, doc):
    """check if at least one word of the document is in the
    word2vec dictionary"""
    return not all(word not in word2vec_model.vocab for word in doc)
%%time


AYE = 1
NAY = -1

vote_matrix_all = {}
X_seq_all = {}
word_index_all = {}
X_train_tf_all = {}
X_train_counts_all = {}
X_emb_all = {}
legistlator_dict_all = {}


grouped_congress = df_bills.groupby('congress')

feature_all = {}
y_all = {}


for name, group in grouped_congress:
    print('Processing congress', name)
    print('congress shape', group.shape)
    
    df_votes_filtered = df_votes[df_votes['congress'] == name]
    
    num_legistlators = len(df_votes_filtered['name'].unique())
    print('number of legistlators', num_legistlators)
    vote_matrix = np.zeros((group.shape[0], num_legistlators))
    print('vote_matrix.shape', vote_matrix.shape)
    
    # Creat legistor dict mapping!
    legistlator_dict = {}
    i = 0
    for legitslator in df_votes_filtered['name'].unique():
        legistlator_dict[legitslator] = i
        i += 1
    
    legistlator_dict_all[name] = {v: k for k, v in legistlator_dict.items()} 
    
    ########
#     majority, party, vote
#     print(pd.get_dummies(df_votes_filtered['majority']).shape)
#     print(pd.get_dummies(df_votes_filtered['party']).shape)
#     feature_all[name] = np.hstack((pd.get_dummies(df_votes_filtered['majority']), 
#                                   pd.get_dummies(df_votes_filtered['party'])))
#     print(preprocessing.OneHotEncoder().fit_transform(df_votes_filtered['majority']))
    
    
#     print((df_votes_filtered['vote'].value_counts()))

#     y_all[name] = df_votes_filtered['vote']
    ########
        
#     print(legistlator_dict)
    print('Processing congress votes')
    group.reset_index(inplace=True)
    
    vote_feature = []
    # Process Vote matrix here
    for index, row in tqdm(group.iterrows()):
        
        df_filtered = df_votes_filtered[df_votes_filtered['legis_num'] == row['legis_num']]

        for _, vote in df_filtered.iterrows():
            vote_feature.append({'majority' : vote['majority'], 
                         'party' : vote['party'],
                         'sponser' : row['sponsor_party'],
                         'y' :  vote['vote']})
            if vote['vote'] == 'Yea':
                vote_matrix[index, legistlator_dict[vote['name']]] = AYE
            elif vote['vote'] == 'Nay':
                vote_matrix[index, legistlator_dict[vote['name']]] = NAY

    # Process bill Representation Here
    print('processing congress bills')
    X_seq, word_index, x_emb, X_train_tf, X_train_counts = process_doc(group['billText'].apply(str))

    feature_all[name] = vote_feature
    vote_matrix_all[name] = vote_matrix
    X_seq_all[name] = X_seq
    word_index_all[name] = word_index
    X_emb_all[name] = x_emb
    X_train_tf_all[name] = X_train_tf
    X_train_counts_all[name] = X_train_counts
    
    print('*' * 50)
    break
    
# print('feature_all', feature_all[106].shape)
# print('y_all', y_all[106].shape)
    
print('vote_matrix_all.shape', vote_matrix_all[106].shape)
np.save('data/vote_matrix_all.npy', vote_matrix_all)
np.save('data/X_seq_all.npy', X_seq_all)
np.save('data/X_word_index_all.npy', word_index_all)
np.save('data/X_train_tf_all.npy', X_train_tf_all)
np.save('data/X_train_counts_all.npy', X_train_counts_all)
np.save('data/X_emb_all.npy', X_emb_all)
np.save('data/legistlator_all.npy', legistlator_dict_all)

# print(feature_all[106].shape)
df_train = pd.DataFrame(feature_all[106])
df_train.tail()
%%time


# PARTY LINE EXPERIMENTS:

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn import preprocessing
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis


encoder = preprocessing.LabelEncoder()

print(encoder.fit_transform(df_train['majority']).shape)
X = np.hstack((encoder.fit_transform(df_train['majority']), 
               encoder.fit_transform(df_train['party']), 
               encoder.fit_transform(df_train['sponser'])))

# print(X)
y = df_train['y'].tolist()
le = preprocessing.LabelEncoder()
le.fit(y)
print(le.classes_)
y = le.transform(y)
print(X[:10])
print(y[:10])

print(X.shape)
print(y.shape)

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)

print('X_train', X_train.shape)
print('y_train', y_train.shape)


# classifiers = [
#     KNeighborsClassifier(3),
#     SVC(kernel="linear", C=0.025),
#     SVC(gamma=2, C=1),
#     GaussianProcessClassifier(1.0 * RBF(1.0)),
#     DecisionTreeClassifier(max_depth=5),
#     RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
#     MLPClassifier(alpha=1),
#     AdaBoostClassifier(),
#     GaussianNB(),
#     QuadraticDiscriminantAnalysis()]


# clf = SVC(kernel="linear", C=0.025)
# clf = LinearRegression()
clf = MLPClassifier(alpha=1)
# clf = AdaBoostClassifier()
clf.fit(X_train, y_train)

y_predict = clf.predict(X_test)
print('score', clf.score(X_test, y_predict))

regression_model_mse = mean_squared_error(y_test, y_predict)
print('accuracy', accuracy_score(y_test, y_predict))

print('model_mse', regression_model_mse)

print(confusion_matrix(y_test, y_predict))
print(classification_report(y_test, y_predict, target_names=le.classes_))

print(vote_matrix_all[106].shape)
print(X_seq_all[106].shape)
# print(word_index_all[106].shape)
print(X_train_tf_all[106].shape)
print(X_train_counts_all[106].shape)
print(X_emb_all[106].shape)
congress = 115
x = X_emb_all[congress]
print('x', x.shape)
df_congress = df_bills[df_bills['congress'] == congress]
y = df_congress['party'].values
print(y.shape)

from sklearn import preprocessing
le = preprocessing.LabelEncoder()
le.fit(y)
print(le.classes_)
y = le.transform(y)
print(y)

colors = ['blue', 'red']
# Pretty Useless :) 
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
x_pca = pca.fit_transform(x)

plt.figure(1, figsize=(20, 20),)
plt.scatter(x_pca[:, 0], x_pca[:, 1], s=100, c=y, cmap=matplotlib.colors.ListedColormap(colors), alpha=0.6)

from sklearn.manifold import TSNE
X_tsne = TSNE(n_components=2, verbose=2).fit_transform(x)

plt.figure(1, figsize=(20, 20),)
plt.scatter(X_tsne[:, 0], X_tsne[:, 1], s=100, c=y, cmap=matplotlib.colors.ListedColormap(colors), alpha=0.6)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup