<jupyter_start><jupyter_text>In this notebook we will use the previously generated plot similarities to remake the paper. The codes from the last notebook are compiled in the Featurizer class in the featurizer.py file.<jupyter_code>from featurizer import Featurizer
cf = Featurizer(plot_vectorizer = 'binary', tokenizer = None, lda = False)
cf.load_train('data.json')
cf.find_movie('Tokyo Drift')<jupyter_output><empty_output><jupyter_text>In order to make a linear combination of all features we have (as the paper uses), we need to do a linear regression on some training data.<jupyter_code>from featurizer import Featurizer
import numpy as np
from scipy.sparse import issparse

def first_find(title, year = None):
    return cf.find_movie(title, year)[0]

def training_row(A, B, score, featurizer):
    fv = featurizer.single_features(A,B)
    return np.hstack((fv.todense() if issparse(fv) else fv, np.array(score).reshape((1,1))))
import io, json

def get_train_data(path):
    with io.open(path, 'r', encoding='latin-1') as f:
        data = json.load(f)
        return [((movie['movie1']['title'],movie['movie1']['year']), \
                 (movie['movie2']['title'],movie['movie2']['year']), movie['score']) \
                for movie in data]

training_data = get_train_data('training.json')
def training_vectors(training_data, featurizer):
    v = [training_row(A, B, score, featurizer) for A, B, score in training_data]
    v = np.vstack(v)
    v = np.hsplit(v,[-1])
    return v[0], v[1]

A, b = training_vectors(training_data, cf)
print(A.shape)
print(b.shape)
from sklearn.linear_model import LinearRegression

def lin_reg(A, b):
    lr = LinearRegression().fit(A,b)
    r2 = lr.score(A,b)
    coef = lr.coef_
    return r2, coef

r2, coef = lin_reg(A,b)
print('R^2 value:',r2)
del(cf)<jupyter_output><empty_output><jupyter_text>Sample runs.<jupyter_code>def linear_test(training_data, path, plot_vectorizer = 'count', tokenizer = None, lda = False, use_genre_vecs = False):
    cf = Featurizer(plot_vectorizer = plot_vectorizer, tokenizer = tokenizer, lda = lda, use_genre_vecs = use_genre_vecs)
    cf.load_train(path)
    A, b = training_vectors(training_data, cf)
    r2, coef = lin_reg(A, b)
    print('R^2:',r2)
    print('Similar to the Avengers:',cf.similar_movies(coef, ('The Avengers', '2012'), movies = None, n = 15))
    print('Similar to Burlesque:',cf.similar_movies(coef, ('Burlesque','2010/I'), movies = None, n = 15))
    print('Similar to Gladiator:',cf.similar_movies(coef, ('Gladiator','2000'), movies = None, n = 15))
    print('Similar to Tokyo Drift:',cf.similar_movies(coef, ('The Fast and the Furious: Tokyo Drift','2006'), movies = None, n = 15))
    
#linear_test(training_data, 'binary', tokenizer = None, use_genre_vecs = True)
plot_vects = ['count','binary','tfidf']
tokenizers = [None]
genre_vecs = [False, True]
datas = ['data.json', 'ne_data.json']

for gv in genre_vecs:
    for data in datas:
        for tokenizer in tokenizers:
            for plot_vect in plot_vects:
                print('Plot vectorizer:',plot_vect)
                print('Tokenizer:',tokenizer)
                print('Using genre vectors:', gv)
                print('Using data:', data)
                linear_test(training_data, path = data, plot_vectorizer = plot_vect, \
                            tokenizer = tokenizer, use_genre_vecs = gv)
plot_vects = ['count','binary','tfidf']
tokenizers = [None]
genre_vecs = [False, True]
datas = ['data.json', 'ne_data.json']

for data in datas:
    for tokenizer in tokenizers:
        for gv in genre_vecs:
            for plot_vect in plot_vects:
                print('Plot vectorizer:',plot_vect)
                print('Tokenizer:',tokenizer)
                print('Using genre vectors:', gv)
                print('Using data:', data)
                linear_test(training_data, path = data, plot_vectorizer = plot_vect, \
                            tokenizer = tokenizer, use_genre_vecs = gv)
plot_vects = ['count','binary','tfidf']
tokenizers = [None]
genre_vecs = [True]
datas = ['ne_data.json','data.json']

for data in datas:
    for tokenizer in tokenizers:
        for gv in genre_vecs:
            for plot_vect in plot_vects:
                print('Plot vectorizer:',plot_vect)
                print('Tokenizer:',tokenizer)
                print('Using genre vectors:', gv)
                print('Using data:', data)
                linear_test(training_data, path = data, plot_vectorizer = plot_vect, \
                            tokenizer = tokenizer, use_genre_vecs = gv)<jupyter_output>Plot vectorizer: count
Tokenizer: None
Using genre vectors: True
Using data: ne_data.json
R^2: 0.668943685437
Similar to the Avengers: [('Songbirth', '2015'), ('The Godfather Family: A Look Inside', '1990'), ('Songbook', '2008'), ('Maple Flavour Films', '2008'), ('Nigger', '2012'), ('Rock & Roll', '1995'), ('Pick Up the Mic', '2006'), ("The Making of Ste<NAME>'s 'Jaws'", '1995'), ('The Pod', '2014'), ('Misconception', '2009'), ('Trinity and Beyond: The Atomic Bomb Movie', '1995'), ('The Summer of Massacre', '2012'), ('The Celluloid Closet', '1995'), ('Tardisodes', '2006'), ('The First Measured Century', '2000')]
Similar to Burlesque: [('The Godfather Family: A Look Inside', '1990'), ('Maple Flavour Films', '2008'), ('Nigger', '2012'), ('The Celluloid Closet', '1995'), ('Tardisodes', '2006'), ('The Summer of Massacre', '2012'), ('Pick Up the Mic', '2006'), ('Rock & Roll', '1995'), ('The First Measured Century', '2000'), ('The Pod', '2014'), ('Misconception', '2009'), ("The Making[...]
StarcoderdataJupyterStructuredCleanDedup