<jupyter_start><jupyter_text>Веса инициализируются в момент опсиания архитектуры сети. Все возможные варианты инициализации весов приведены в официальной [документации](https://pytorch.org/docs/stable/nn.init.html). В примере ниже веса инициализируются методом Ксавье, что соотвествует использованию `tanh` в качестве функции активации<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')
%cd /content/gdrive/My Drive/HSE_DL_2021/04_week
from IPython import display
display.Image('images/xavier.png')
'''
STEP 3: CREATE MODEL CLASS
'''
class FeedforwardNeuralNetModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim, act_type='relu', init_type='he'):
        super(FeedforwardNeuralNetModel, self).__init__()
        # Linear function
        self.fc1 = nn.Linear(input_dim, hidden_dim) 
        
        # Non-linearity
        if act_type == 'tanh':
            self.activation = nn.Tanh()
        elif act_type == 'relu':
            self.activation = nn.ReLU()
        elif act_type == 'sigmoid':
            self.activation = nn.Sigmoid()
        else:
            print('This kind of activation is not supported in this net')

        # Linear function (readout)
        self.fc2 = nn.Linear(hidden_dim, output_dim)  
        
        # Linear weight, W,  Y = WX + B
        if init_type == 'xavier':
            nn.init.xavier_normal_(self.fc1.weight)
            nn.init.xavier_normal_(self.fc2.weight)
        elif init_type == 'he':
            nn.init.kaiming_normal_(self.fc1.weight)
            nn.init.kaiming_normal_(self.fc2.weight)
        else:
            print('This kind of initialization is not supported in this net')

    def forward(self, x):
        # Linear function
        out = self.fc1(x)
        # Non-linearity
        out = self.activation(out)
        # Linear function (readout)
        out = self.fc2(out)
        return out
a = torch.empty(3,3)
print(a)
nn.init.xavier_normal_(a)
'''
STEP 4: INSTANTIATE BASE PARAMETERS
'''
input_dim = 28*28
hidden_dim = 100
output_dim = 10

model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)
'''
STEP 5: INSTANTIATE LOSS CLASS
'''
criterion = nn.CrossEntropyLoss()
'''
STEP 6: INSTANTIATE OPTIMIZER CLASS
'''
learning_rate = 0.1

optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
'''
STEP 7: INSTANTIATE STEP LEARNING SCHEDULER CLASS
'''
# step_size: at how many multiples of epoch you decay
# step_size = 1, after every 1 epoch, new_lr = lr*gamma 
# step_size = 2, after every 2 epoch, new_lr = lr*gamma 

# gamma = decaying factor
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

'''
STEP 8: TRAIN THE MODEL
'''
def model_train(model, criterion, optimizer, scheduler, train_loader, test_loader):
    iter = 0
    for epoch in range(num_epochs):
        # Print Learning Rate
        print('Epoch:', epoch,'LR:', scheduler.get_last_lr()[0])
        for i, (images, labels) in enumerate(train_loader):
            # Load images as tensors with gradient accumulation abilities
            images = images.view(-1, 28*28).requires_grad_()

            # Clear gradients w.r.t. parameters
            optimizer.zero_grad()

            # Forward pass to get output/logits
            outputs = model(images)

            # Calculate Loss: softmax --> cross entropy loss
            loss = criterion(outputs, labels)

            # Getting gradients w.r.t. parameters
            loss.backward()

            # Updating parameters
            optimizer.step()

            iter += 1

            if iter % 500 == 0:
                # Calculate Accuracy         
                correct = 0
                total = 0
                # Iterate through test dataset
                for images, labels in test_loader:
                    # Load images to a Torch Variable
                    images = images.view(-1, 28*28)

                    # Forward pass only to get logits/output
                    outputs = model(images)

                    # Get predictions from the maximum value
                    _, predicted = torch.max(outputs.data, 1)

                    # Total number of labels
                    total += labels.size(0)

                    # Total correct predictions
                    correct += (predicted.type(torch.FloatTensor).cpu() == labels.type(torch.FloatTensor)).sum()

                accuracy = 100. * correct.item() / total

                # Print Loss
                print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))

        # Decay Learning Rate
        scheduler.step()<jupyter_output><empty_output><jupyter_text>ReLU + Xavier<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='relu', init_type='xavier')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.10658138990402222. Accuracy: 95.96
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.0844404399394989. Accuracy: 96.86
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.05578886345028877. Accuracy: 97.24
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.06414199620485306. Accuracy: 97.67
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.0548870787024498. Accuracy: 97.78
Iteration: 3000. Loss: 0.07690365612506866. Accuracy: 97.71<jupyter_text>Sigmoid + Xavier<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='sigmoid', init_type='xavier')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.19519919157028198. Accuracy: 92.94
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.11756786704063416. Accuracy: 94.34
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.07408924400806427. Accuracy: 95.28
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.11367665231227875. Accuracy: 95.92
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.08256454765796661. Accuracy: 96.21
Iteration: 3000. Loss: 0.10840769857168198. Accuracy: 96.62<jupyter_text>Tanh + Xavier<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='tanh', init_type='xavier')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.17831073701381683. Accuracy: 95.26
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.07358894497156143. Accuracy: 96.66
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.06701365113258362. Accuracy: 97.15
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.027918489649891853. Accuracy: 97.12
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.02466857247054577. Accuracy: 97.59
Iteration: 3000. Loss: 0.02207859791815281. Accuracy: 97.56<jupyter_text>ReLU + He<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='relu', init_type='he')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.16296333074569702. Accuracy: 96.11
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.1221468523144722. Accuracy: 96.73
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.20114415884017944. Accuracy: 96.77
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.08712247759103775. Accuracy: 97.49
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.10436874628067017. Accuracy: 97.46
Iteration: 3000. Loss: 0.03802387788891792. Accuracy: 97.72<jupyter_text>Sigmoid + He<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='sigmoid', init_type='he')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.39061737060546875. Accuracy: 93.01
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.16960738599300385. Accuracy: 94.54
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.21181795001029968. Accuracy: 95.28
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.13369661569595337. Accuracy: 96.15
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.11100592464208603. Accuracy: 96.39
Iteration: 3000. Loss: 0.1432812362909317. Accuracy: 96.85<jupyter_text>Tanh + He<jupyter_code>model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim, act_type='tanh', init_type='he')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
scheduler = StepLR(optimizer, step_size=1, gamma=0.96)

model_train(model, criterion, optimizer, scheduler, train_loader, test_loader)<jupyter_output>Epoch: 0 LR: 0.1
Iteration: 500. Loss: 0.14771655201911926. Accuracy: 95.68
Epoch: 1 LR: 0.096
Iteration: 1000. Loss: 0.146752268075943. Accuracy: 96.6
Epoch: 2 LR: 0.09215999999999999
Iteration: 1500. Loss: 0.045555971562862396. Accuracy: 97.21
Epoch: 3 LR: 0.08847359999999999
Iteration: 2000. Loss: 0.04142989218235016. Accuracy: 97.42
Epoch: 4 LR: 0.08493465599999998
Iteration: 2500. Loss: 0.03965333476662636. Accuracy: 97.4
Iteration: 3000. Loss: 0.020156938582658768. Accuracy: 97.38<jupyter_text>[Источник](https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/weight_initialization_activation_functions/) ноутбука<jupyter_code><jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup