<jupyter_start><jupyter_text>CHAPTER 5 TENSORFLOW<jupyter_code>import tensorflow as tf
from tensorflow import keras 
from tensorflow.keras.utils import to_categorical
import numpy as np
import logging

tf.get_logger().setLevel(logging.ERROR)
tf.random.set_seed(7)

EPOCHS = 20
BATCH_SIZE = 1
# Load training and test datasets.
mnist = keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

# Standardize the data
mean = np.mean(train_images)
stddev = np.std(train_images)
train_images = (train_images - mean) / stddev
test_images = (test_images - mean) / stddev

# One-hot encode labels
train_labels = to_categorical(train_labels, num_classes=10)
test_labels = to_categorical(test_labels, num_classes=10)
# Check the encoding
train_labels[1]<jupyter_output><empty_output><jupyter_text>The Network<jupyter_code># Initialize weights
initializer = keras.initializers.RandomUniform(minval=-0.1, maxval=0.1)

# A sequential model
# 784 inputs
# Two dense (fully connected) layers with 25 neurons and 10 neurons
# Hidden Layer: Activation (tanh)
# Output Layer: Activation (Logistic function)
model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28,28)), # 784
    keras.layers.Dense(25, activation='tanh',kernel_initializer=initializer,bias_initializer='zeros'), 
    keras.layers.Dense(10, activation='sigmoid', kernel_initializer=initializer, bias_initializer='zeros')
])
model.summary()
# Using stochastic gradient descent (SDG)
# Learning rate: 0.01 
# Loss function: MSE
opt = keras.optimizers.SGD(learning_rate=0.01)
model.compile(loss='mean_squared_error', optimizer=opt, metrics=['accuracy'])

# Model training
# Epochs: 20
# Update weights after each example or batch of examples.
history = model.fit(
    train_images, train_labels,
    validation_data=(test_images, test_labels),
    epochs=EPOCHS, batch_size=BATCH_SIZE,
    verbose=2, shuffle=True
)
### Configuration 5 Example
# Enploying different techniques to counter vanishing and or exploding gradients.
model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(25, activation='relu', kernel_initializer='he_normal', bias_initializer='zeros'),
    keras.layers.Dense(10, activation='softmax', kernel_initializer='glorot_uniform', bias_initializer='zeros')
])

# Faster optimizers
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# Model training
history = model.fit(
    train_images, train_labels,
    validation_data=(test_images, test_labels),
    epochs=20, batch_size=64,
    verbose=2, shuffle=True
)<jupyter_output>Epoch 1/20
938/938 - 1s - loss: 0.3473 - accuracy: 0.8958 - val_loss: 0.2153 - val_accuracy: 0.9389
Epoch 2/20
938/938 - 1s - loss: 0.2007 - accuracy: 0.9424 - val_loss: 0.1827 - val_accuracy: 0.9468
Epoch 3/20
938/938 - 1s - loss: 0.1685 - accuracy: 0.9510 - val_loss: 0.1649 - val_accuracy: 0.9498
Epoch 4/20
938/938 - 1s - loss: 0.1453 - accuracy: 0.9574 - val_loss: 0.1532 - val_accuracy: 0.9546
Epoch 5/20
938/938 - 1s - loss: 0.1331 - accuracy: 0.9604 - val_loss: 0.1405 - val_accuracy: 0.9592
Epoch 6/20
938/938 - 1s - loss: 0.1215 - accuracy: 0.9640 - val_loss: 0.1444 - val_accuracy: 0.9580
Epoch 7/20
938/938 - 1s - loss: 0.1122 - accuracy: 0.9663 - val_loss: 0.1436 - val_accuracy: 0.9581
Epoch 8/20
938/938 - 1s - loss: 0.1049 - accuracy: 0.9688 - val_loss: 0.1342 - val_accuracy: 0.9622
Epoch 9/20
938/938 - 1s - loss: 0.0983 - accuracy: 0.9704 - val_loss: 0.1400 - val_accuracy: 0.9605
Epoch 10/20
938/938 - 1s - loss: 0.0923 - accuracy: 0.9714 - val_loss: 0.1419 - val_accuracy: 0.9617[...]<jupyter_text>PYTORCH<jupyter_code>import torch
import torch.nn as nn 
import torchvision
import torchvision.transforms as T
from torchvision.datasets import MNIST 
from torch.utils.data import DataLoader
import numpy as np 
torch.manual_seed(7)
# Set 'GPU' or 'CPU' as the base device for training
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
EPOCHS = 20
# BATCH_SIZE = 1
BATCH_SIZE = 64
# Load the training set into a single batch to compute mean and stddev.
transform = T.Compose([T.ToTensor()])
# Helper MNIST object for calculating the mean and stddev using the batch_size the size of the train dataset.
trainset = MNIST(root='./data/pt_mnist', train=True, download=True, transform=transform)
trainloader = DataLoader(trainset, batch_size=len(trainset), shuffle=True)
# Load all the data to calculate the mean and standard deviation to scale and center the dataset.
data = next(iter(trainloader))
nn.functional.one_hot(data[1][0], num_classes=10)
mean = data[0].mean()
stddev = data[0].std()
# Standardize data on loading.
transform = T.Compose([
    T.ToTensor(),
    T.Normalize(mean=mean, std=stddev)
])
trainset = MNIST(root="./data/pt_mnist", train=True, download=True, transform=transform)
testset = MNIST(root="./data/pt_mnist", train=True, download=True, transform=transform)
# A sequential model
# 784 inputs
# Two dense (fully connected) layers with 25 neurons and 10 neurons
# Hidden Layer: Activation (tanh)
# Output Layer: Activation (Logistic function)
model = nn.Sequential(
    nn.Flatten(),
    nn.Linear(784, 25), 
    nn.Tanh(), 
    nn.Linear(25, 10),
    nn.Sigmoid()
)

# Initialize weights
for module in model.modules():
    if isinstance(module, nn.Linear):
        # Initialize layer weights
        nn.init.uniform_(module.weight, a=-0.1, b=0.1)
        # Initialize bias weights
        nn.init.constant_(module.bias, 0.0)
model
### TRAINING
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss_function = nn.MSELoss()

# Set to the supported device
model.to(device)

# Test and train loader
trainloader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)
testloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=True)

# Train the model
for i in range(EPOCHS):
    model.train() # Training mode 
    train_loss = 0.0 
    train_correct = 0
    train_batches = 0
    for inputs, targets in trainloader:
        # Move to the same device as model.
        one_hot_targets = nn.functional.one_hot(targets, num_classes=10).float()
        inputs, targets, one_hot_targets = inputs.to(device), targets.to(device), one_hot_targets.to(device)
        # Zero the parameter gradients.
        optimizer.zero_grad()
        # Forward pass
        outputs = model(inputs)
        loss = loss_function(outputs, one_hot_targets)
        # Accumulate metrics 
        _, indices = torch.max(outputs.data, 1)
        train_correct += (indices == targets).sum().item()
        train_batches += 1
        train_loss += loss.item()
        # Backward pass and update
        loss.backward()
        optimizer.step()

    train_loss = train_loss / train_batches
    train_acc = train_correct / (train_batches * BATCH_SIZE)

    # Evaluate the model
    model.eval() # Set model to inference mode.
    test_loss = 0.0
    test_correct = 0
    test_batches = 0
    for inputs, targets in testloader:
        one_hot_targets = nn.functional.one_hot(targets, num_classes=10)
        inputs, targets, one_hot_targets = inputs.to(device), targets.to(device), one_hot_targets.to(device)
        outputs = model(inputs)
        loss = loss_function(outputs, one_hot_targets)
        _, indices = torch.max(outputs, 1)
        test_correct += (indices == targets).sum().item()
        test_batches += 1
        test_loss += loss.item()

    test_loss = test_loss / test_batches 
    test_acc = test_correct / (test_batches * BATCH_SIZE)

    print(f'Epoch {i+1}/{EPOCHS} loss: {train_loss:.4f} - acc: {train_acc:0.4f} - val_loss: {test_loss:.4f} - val_acc: {test_acc:0.4f}')
### Configuration 5 Example
model = nn.Sequential(
    nn.Flatten(), 
    nn.Linear(784, 25),
    nn.ReLU(),
    nn.Linear(25, 10)
)

# Retrieve layers for custom weight initialization
layers = next(model.modules())
hidden_layer = layers[1]
output_layer = layers[3]

# Kaiming (He) intialization
nn.init.kaiming_normal_(hidden_layer.weight)
nn.init.constant_(hidden_layer.bias, 0.0)

# Xavier (Glorot) initialization
nn.init.xavier_uniform_(output_layer.weight)
nn.init.constant_(output_layer.bias, 0.0)
### TRAINING (Configuration 5)
optimizer = torch.optim.Adam(model.parameters())
loss_function = nn.CrossEntropyLoss()

# Set to the supported device
model.to(device)

# Test and train loader
trainloader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)
testloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=True)

# Train the model
for i in range(EPOCHS):
    model.train() # Training mode 
    train_loss = 0.0 
    train_correct = 0
    train_batches = 0
    for inputs, targets in trainloader:
        # Move to the same device as model.
        # one_hot_targets = nn.functional.one_hot(targets, num_classes=10).float()
        # Cross entropy does not need one-hot targets in Pytorch.
        inputs, targets = inputs.to(device), targets.to(device) # one_hot_targets.to(device)
        # Zero the parameter gradients.
        optimizer.zero_grad()
        # Forward pass
        outputs = model(inputs)
        loss = loss_function(outputs, targets)
        # Accumulate metrics 
        _, indices = torch.max(outputs.data, 1)
        train_correct += (indices == targets).sum().item()
        train_batches += 1
        train_loss += loss.item()
        # Backward pass and update
        loss.backward()
        optimizer.step()

    train_loss = train_loss / train_batches
    train_acc = train_correct / (train_batches * BATCH_SIZE)

    # Evaluate the model
    model.eval() # Set model to inference mode.
    test_loss = 0.0
    test_correct = 0
    test_batches = 0
    for inputs, targets in testloader:
        # one_hot_targets = nn.functional.one_hot(targets, num_classes=10)
        inputs, targets = inputs.to(device), targets.to(device)
        outputs = model(inputs)
        loss = loss_function(outputs, targets)
        _, indices = torch.max(outputs, 1)
        test_correct += (indices == targets).sum().item()
        test_batches += 1
        test_loss += loss.item()

    test_loss = test_loss / test_batches 
    test_acc = test_correct / (test_batches * BATCH_SIZE)

    print(f'Epoch {i+1}/{EPOCHS} loss: {train_loss:.4f} - acc: {train_acc:0.4f} - val_loss: {test_loss:.4f} - val_acc: {test_acc:0.4f}')
### PLOT ACTIVATION FUNCTIONS
import matplotlib.pyplot as plt
tanh = lambda z: np.tanh(z)
sigmoid = lambda z: np.exp(z) / (np.exp(z) + 1)
z = np.linspace(-4, 4, num=9)
plt.figure(figsize=(8,5))
plt.plot(z, tanh(z), label="y = tanh(z)")
plt.plot(z, sigmoid(z), label="$y = e^z/(e^z+1)$")
plt.xlabel("z")
plt.ylabel("y")
plt.grid(True)
plt.legend()
### Cross-Entropy function
def cross_entropy(y_pred, y_true):
    if y_true == 1.0:
        return -np.log(y_pred)
    else:
        return -np.log(1.0 - y_pred)

cross_ent = lambda y,y_hat: -np.log(y_hat) if y==1.0 else -np.log(1 - y_hat)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup