<jupyter_start><jupyter_text>Simple baseline code<jupyter_code>y_pred = np.full(y_test.shape, y_train.mean())<jupyter_output><empty_output><jupyter_text>Metric<jupyter_code>mean_squared_error(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>Add features<jupyter_code>X_train.text = X_train.text.str.replace('[^\w\s]',' ')
X_test.text = X_test.text.str.replace('[^\w\s]',' ')
X_train.parent_text = X_train.parent_text.str.replace('[^\w\s]',' ')
X_test.parent_text = X_test.parent_text.str.replace('[^\w\s]',' ')
X_train = '"' + X_train + '"'
X_test = '"' + X_test + '"'<jupyter_output><empty_output><jupyter_text>Save data for training<jupyter_code>pd.concat([X_train, y_train], axis=1).sample(n=100000).to_csv('./train.csv', index=False, sep='\t')
pd.concat([X_test, y_test], axis=1).to_csv('./valid.csv', index=False, sep='\t')
del X_train, X_test, df_train, df_test, df<jupyter_output><empty_output><jupyter_text>PyTorch Bag of words<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F

from torchtext.data import Field, BucketIterator, TabularDataset, Iterator, LabelField
from torch.utils.data import DataLoader
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from torch import optim
import tqdm
import torchtext
torch.cuda.is_available()
BATCH_SIZE = 400
MAX_VOCAB_SIZE = 50000
EMBEDDING_SIZE = 50
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# device = 'cpu'
def add_bigrams(tokens):
    bigrams = [f'{current_word} {next_word}' for current_word, next_word in 
                          zip(tokens, tokens[1:])]
    tokens.extend(bigrams)
    return tokens
TEXT = Field(tokenize="spacy",
             preprocessing=add_bigrams,
             batch_first=True,
             include_lengths=True)
SCORE = LabelField(dtype=torch.float, 
                   use_vocab=False)

train_data, valid_data = TabularDataset.splits(
        path='./', train='train.csv', validation='valid.csv', 
        format='tsv', csv_reader_params={"escapechar": '&'},
        fields={
                'text': ('text', TEXT),
                'parent_text': ('parent_text', TEXT),
                'scaled_score': ('score', SCORE)
              }
)
TEXT.build_vocab(train_data, 
                 max_size=MAX_VOCAB_SIZE,
                 vectors=f'glove.6B.{EMBEDDING_SIZE}d',
                 unk_init=torch.Tensor.normal_,
                 vectors_cache='/content/gdrive/My Drive/cache')

TEXT.vocab.vectors[TEXT.vocab.stoi['<unk>']] = 0
TEXT.vocab.vectors[TEXT.vocab.stoi['<pad>']] = 0
train_iter = BucketIterator(
        train_data,
        batch_size=BATCH_SIZE,
        sort_key=lambda x: len(x.text),
        sort_within_batch=True,
        device=device
)

valid_iter = BucketIterator(
        valid_data,
        batch_size=BATCH_SIZE,
        sort=False,
        device=device
)
def train(model, iterator, optimizer, criterion):
    epoch_loss = 0    
    model.train()
    
    for batch in tqdm.tqdm_notebook(iterator):
        optimizer.zero_grad()
        predictions = model(batch.text[0], batch.text[1], batch.parent_text[0], batch.parent_text[1])
        loss = criterion(predictions, batch.score)
        
        loss.backward()
        optimizer.step()
        
        epoch_loss += loss.item()
        
    return epoch_loss / len(iterator)

def evaluate(model, iterator, criterion):
    epoch_loss = 0
    model.eval()
    with torch.no_grad():
        for batch in iterator:
            predictions = model(batch.text[0], batch.text[1], batch.parent_text[0], batch.parent_text[1])
            loss = criterion(predictions, batch.score)
            epoch_loss += loss.item()

    return epoch_loss / len(iterator)

def get_preds(model, iterator):
    predictions = []
    model.eval()
    with torch.no_grad():
        for batch in iterator:
            batch_predictions = model(batch.text[0], batch.text[1], batch.parent_text[0], batch.parent_text[1])
            predictions.append(batch_predictions.cpu().numpy())
    return np.array(predictions).flatten()
import time
def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs
    
def train_loop(model, n_epochs, init_lr, decay_rate=0.1, decay_steps=1, save_name=None):
    optimizer = optim.Adam(model.parameters(), lr=init_lr)
    criterion = nn.MSELoss()
    scheduler = optim.lr_scheduler.StepLR(optimizer, decay_steps, decay_rate) 

    best_valid_loss = float('inf')

    for epoch in range(n_epochs):    
        start_time = time.time()
        
        train_loss = train(model, train_iter, optimizer, criterion)
        valid_loss = evaluate(model, valid_iter, criterion)
        
        scheduler.step()
        
        end_time = time.time()
        epoch_mins, epoch_secs = epoch_time(start_time, end_time)
        
        if valid_loss < best_valid_loss and save_name is not None:
            best_valid_loss = valid_loss
            torch.save(model.state_dict(), save_name)
        
        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
        print(f'\tTrain Loss: {train_loss:.3f} | Val. Loss: {valid_loss:.3f}')
class EmbeddingBagModel(nn.Module):
    def __init__(self, 
                 vocab_size: int, 
                 embedding_dim: int,
                 text_fc_size: int, 
                 parent_fc_size: int,
                 padding_idx: int):
        super().__init__()
        self.padding_idx = padding_idx
        # EmbeddingBag lacks padding_idx so we have to use if in forward
        self.emb_bag = nn.EmbeddingBag(vocab_size, embedding_dim, mode='sum')
        self.text_emb_bn = nn.BatchNorm1d(embedding_dim)
        self.parent_emb_bn = nn.BatchNorm1d(embedding_dim)

        self.text_fc = nn.Linear(embedding_dim, text_fc_size)
        self.parent_fc = nn.Linear(embedding_dim, parent_fc_size)

        self.output_fc = nn.Linear(text_fc_size + parent_fc_size, 1)
        
    def init_emb_from_vocab(self, vocab: torchtext.vocab.Vocab, freeze=False):
        if not hasattr(vocab, 'vectors'):
            raise ValueError("To init embedding from vocab it needs to contain pretrained vectors")
        self.emb_bag.weight.data.copy_(vocab.vectors)
        if freeze:
            self.freeze_embedding()
        else:
            self.unfreeze_embedding()
        
    def forward(self, text, text_lengths, parent_text, parent_lengths):
        self.emb_bag.weight.data[self.padding_idx] = 0
        text = self.emb_bag(text) / text_lengths.view(-1, 1)
        text = self.text_emb_bn(text)
        text = F.relu(self.text_fc(text))

        parent_text = self.emb_bag(parent_text) / parent_lengths.view(-1, 1)
        parent_text = self.parent_emb_bn(parent_text)
        parent_text = F.relu(self.parent_fc(parent_text))

        joint = torch.cat([text, parent_text], dim=1)
        return self.output_fc(joint).squeeze(1)

    def freeze_embedding(self):
      self.emb_bag.weight.requires_grad = False

    def unfreeze_embedding(self):
      self.emb_bag.weight.requires_grad = True
TEXT_FC_SIZE = 200
PARENT_FC_SIZE = 200

model = EmbeddingBagModel(len(TEXT.vocab), EMBEDDING_SIZE, TEXT_FC_SIZE, PARENT_FC_SIZE, TEXT.vocab.stoi['<pad>'])
model.init_emb_from_vocab(TEXT.vocab, freeze=True)

model = model.to(device)
train_loop(model, 10, 1e-2, decay_rate=0.5, decay_steps=3, save_name='/content/gdrive/My Drive/chk/emb_bag.pt')
model.unfreeze_embedding()
train_loop(model, 10, 1e-2, decay_rate=0.5, save_name='/content/gdrive/My Drive/chk/emb_bag.pt')
mean_squared_error(y_test, get_preds(model, valid_iter))
BATCH_SIZE = 400
MAX_VOCAB_SIZE = 20000
EMBEDDING_SIZE = 50
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# device = 'cpu'
def get_min_length_preprocessor(length):
    def pad_to_min_length(tokens):
        if len(tokens) < length:
            tokens.extend(['<pad>'] * (length - len(tokens)))
        return tokens
    return pad_to_min_length

TEXT = Field(tokenize="spacy",
             preprocessing=get_min_length_preprocessor(5),
             batch_first=True,
             include_lengths=True)
SCORE = LabelField(dtype=torch.float, 
                   use_vocab=False)

train_data, valid_data = TabularDataset.splits(
        path='./', train='train.csv', validation='valid.csv', 
        format='tsv', csv_reader_params={"escapechar": '&'},
        fields={
                'text': ('text', TEXT),
                'parent_text': ('parent_text', TEXT),
                'scaled_score': ('score', SCORE)
              }
)

TEXT.build_vocab(train_data, 
                 max_size=MAX_VOCAB_SIZE,
                 vectors=f'glove.6B.{EMBEDDING_SIZE}d',
                 unk_init=torch.Tensor.normal_,
                 vectors_cache='/content/gdrive/My Drive/cache')

TEXT.vocab.vectors[TEXT.vocab.stoi['<unk>']] = 0
TEXT.vocab.vectors[TEXT.vocab.stoi['<pad>']] = 0

train_iter = BucketIterator(
        train_data,
        batch_size=BATCH_SIZE,
        sort_key=lambda x: len(x.text),
        sort_within_batch=True,
        device=device
)

valid_iter = BucketIterator(
        valid_data,
        batch_size=BATCH_SIZE,
        sort=False,
        device=device
)
class CNNModel(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_filters, filter_sizes, hidden_dim, pad_idx):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)
        self.conv_0 = nn.Conv2d(in_channels=1,
                                out_channels=n_filters, 
                                kernel_size=(filter_sizes[0], embedding_dim))
        self.conv_1 = nn.Conv2d(in_channels=1, 
                                out_channels=n_filters, 
                                kernel_size=(filter_sizes[1], embedding_dim))
        self.conv_2 = nn.Conv2d(in_channels=1, 
                                out_channels=n_filters, 
                                kernel_size=(filter_sizes[2], embedding_dim))
        self.fc = nn.Linear(2 * len(filter_sizes) * n_filters, hidden_dim)
        self.output_fc = nn.Linear(hidden_dim, 1)
        
        self.dropout = nn.Dropout(0.5)

    def init_emb_from_vocab(self, vocab: torchtext.vocab.Vocab, freeze=False):
        if not hasattr(vocab, 'vectors'):
            raise ValueError("To init embedding from vocab it needs to contain pretrained vectors")
        self.embedding.weight.data.copy_(vocab.vectors)
        if freeze:
            self.freeze_embedding()
        else:
            self.unfreeze_embedding()

    def forward(self, text, text_lengths, parent_text, parent_lengths):
        text = self.embedding(text).unsqueeze(1)
        text_conv0 = F.relu(self.conv_0(text).squeeze(3))
        text_conv1 = F.relu(self.conv_1(text).squeeze(3))
        text_conv2 = F.relu(self.conv_2(text).squeeze(3))

        text_pooled0 = F.max_pool1d(text_conv0, text_conv0.shape[2]).squeeze(2)
        text_pooled1 = F.max_pool1d(text_conv1, text_conv1.shape[2]).squeeze(2)
        text_pooled2 = F.max_pool1d(text_conv2, text_conv2.shape[2]).squeeze(2)
        
        parent_text = self.embedding(parent_text).unsqueeze(1)
        parent_conv0 = F.relu(self.conv_0(parent_text).squeeze(3))
        parent_conv1 = F.relu(self.conv_1(parent_text).squeeze(3))
        parent_conv2 = F.relu(self.conv_2(parent_text).squeeze(3))

        parent_pooled0 = F.max_pool1d(parent_conv0, parent_conv0.shape[2]).squeeze(2)
        parent_pooled1 = F.max_pool1d(parent_conv1, parent_conv1.shape[2]).squeeze(2)
        parent_pooled2 = F.max_pool1d(parent_conv2, parent_conv2.shape[2]).squeeze(2)

        joint = self.dropout(torch.cat((text_pooled0, text_pooled1, text_pooled2,
                                        parent_pooled0, parent_pooled1, parent_pooled2), dim = 1))
        joint = F.relu(self.fc(joint))
        return self.output_fc(joint).squeeze(1)
        
    def freeze_embedding(self):
      self.embedding.weight.requires_grad = False

    def unfreeze_embedding(self):
      self.embedding.weight.requires_grad = True
JOINT_SIZE = 300
N_FILTERS = 30
FILTER_SIZES = [3,4,5]

model = CNNModel(len(TEXT.vocab), EMBEDDING_SIZE, N_FILTERS, FILTER_SIZES, JOINT_SIZE, TEXT.vocab.stoi['<pad>'])
model.init_emb_from_vocab(TEXT.vocab, freeze=True)

model = model.to(device)
train_loop(model, 20, 1e-2, decay_rate=0.3, decay_steps=4, save_name='/content/gdrive/My Drive/chk/cnn.pt')
model.unfreeze_embedding()
train_loop(model, 10, 1e-3, decay_rate=0.5, decay_steps=3, save_name='/content/gdrive/My Drive/chk/cnn.pt')
train_loop(model, 15, 1e-3, decay_rate=0.5, decay_steps=5, save_name='/content/gdrive/My Drive/chk/cnn.pt')
mean_squared_error(y_test, get_preds(model, valid_iter))<jupyter_output><empty_output><jupyter_text><jupyter_code><jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup