reran and refined old tutorials

.gitignore

@@ -1,4 +1,6 @@
 .idea/
 ML/Pytorch/more_advanced/image_captioning/flickr8k/
 ML/algorithms/svm/__pycache__/utils.cpython-38.pyc
-__pycache__/
+__pycache__/
+*.pth.tar
+*.DS_STORE

ML/Pytorch/Basics/custom_dataset/custom_FCNN.py

@@ -1,131 +0,0 @@
-# Imports
-import os
-from typing import Union
-
-import torch.nn.functional as F  # All functions that don't have any parameters
-import pandas as pd
-import torch
-import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
-import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
-import torchvision
-import torchvision.transforms as transforms  # Transformations we can perform on our dataset
-from pandas import io
-
-# from skimage import io
-from torch.utils.data import (
-    Dataset,
-    DataLoader,
-)  # Gives easier dataset managment and creates mini batches
-import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
-
-
-# Create Fully Connected Network
-class NN(nn.Module):
-    def __init__(self, input_size, num_classes):
-        super(NN, self).__init__()
-        self.fc1 = nn.Linear(input_size, 50)
-        self.fc2 = nn.Linear(50, num_classes)
-
-    def forward(self, x):
-        x = F.relu(self.fc1(x))
-        x = self.fc2(x)
-        return x
-
-
-class SoloDataset(Dataset):
-    def __init__(self, csv_file, root_dir, transform=None):
-        self.annotations = pd.read_csv(csv_file)
-        self.root_dir = root_dir
-        self.transform = transform
-
-    def __len__(self):
-        return len(self.annotations)
-
-    def __getitem__(self, index):
-        x_data = self.annotations.iloc[index, 0:11]
-        x_data = torch.tensor(x_data)
-        y_label = torch.tensor(int(self.annotations.iloc[index, 11]))
-
-        return (x_data.float(), y_label)
-
-
-# Set device
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-# Hyperparameters
-num_classes = 26
-learning_rate = 1e-3
-batch_size = 5
-num_epochs = 30
-input_size = 11
-
-# Load Data
-dataset = SoloDataset(
-    csv_file="power.csv", root_dir="test123", transform=transforms.ToTensor()
-)
-train_set, test_set = torch.utils.data.random_split(dataset, [2900, 57])
-train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)
-test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)
-
-# Model
-model = NN(input_size=input_size, num_classes=num_classes).to(device)
-
-# Loss and optimizer
-criterion = nn.CrossEntropyLoss()
-optimizer = optim.Adam(model.parameters(), lr=learning_rate)
-
-print(len(train_set))
-print(len(test_set))
-# Train Network
-for epoch in range(num_epochs):
-    losses = []
-
-    for batch_idx, (data, targets) in enumerate(train_loader):
-        # Get data to cuda if possible
-        data = data.to(device=device)
-        targets = targets.to(device=device)
-
-        # forward
-        scores = model(data)
-        loss = criterion(scores, targets)
-
-        losses.append(loss.item())
-
-        # backward
-        optimizer.zero_grad()
-        loss.backward()
-
-        # gradient descent or adam step
-        optimizer.step()
-
-    print(f"Cost at epoch {epoch} is {sum(losses) / len(losses)}")
-
-
-# Check accuracy on training to see how good our model is
-def check_accuracy(loader, model):
-    num_correct = 0
-    num_samples = 0
-    model.eval()
-
-    with torch.no_grad():
-        for x, y in loader:
-            x = x.to(device=device)
-            y = y.to(device=device)
-
-            scores = model(x)
-            _, predictions = scores.max(1)
-            num_correct += (predictions == y).sum()
-            num_samples += predictions.size(0)
-
-        print(
-            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct) / float(num_samples) * 100:.2f}"
-        )
-
-    model.train()
-
-
-print("Checking accuracy on Training Set")
-check_accuracy(train_loader, model)
-
-print("Checking accuracy on Test Set")
-check_accuracy(test_loader, model)

ML/Pytorch/Basics/custom_dataset/custom_dataset.py

@@ -6,7 +6,7 @@ label (0 for cat, 1 for dog).
 
 Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
 *    2020-04-03 Initial coding
-
+*    2022-12-19 Updated with better comments, improved code using PIL, and checked code still functions as intended.
 """
 
 # Imports
@@ -17,7 +17,7 @@ import torchvision.transforms as transforms  # Transformations we can perform on
 import torchvision
 import os
 import pandas as pd
-from skimage import io
+from PIL import Image
 from torch.utils.data import (
     Dataset,
     DataLoader,
@@ -35,7 +35,7 @@ class CatsAndDogsDataset(Dataset):
 
     def __getitem__(self, index):
         img_path = os.path.join(self.root_dir, self.annotations.iloc[index, 0])
-        image = io.imread(img_path)
+        image = Image.open(img_path)
         y_label = torch.tensor(int(self.annotations.iloc[index, 1]))
 
         if self.transform:
@@ -50,7 +50,7 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 # Hyperparameters
 in_channel = 3
 num_classes = 2
-learning_rate = 1e-3
+learning_rate = 3e-4
 batch_size = 32
 num_epochs = 10
 
@@ -69,12 +69,19 @@ train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True
 test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)
 
 # Model
-model = torchvision.models.googlenet(pretrained=True)
+model = torchvision.models.googlenet(weights="DEFAULT")
+
+# freeze all layers, change final linear layer with num_classes
+for param in model.parameters():
+    param.requires_grad = False
+
+# final layer is not frozen
+model.fc = nn.Linear(in_features=1024, out_features=num_classes)
 model.to(device)
 
 # Loss and optimizer
 criterion = nn.CrossEntropyLoss()
-optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
 
 # Train Network
 for epoch in range(num_epochs):

ML/Pytorch/Basics/pytorch_bidirectional_lstm.py

@@ -1,15 +1,17 @@
 """
 Example code of a simple bidirectional LSTM on the MNIST dataset.
+Note that using RNNs on image data is not the best idea, but it is a 
+good example to show how to use RNNs that still generalizes to other tasks.
 
 Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
 *    2020-05-09 Initial coding
+*    2022-12-16 Updated with more detailed comments, docstrings to functions, and checked code still functions as intended.
 
 """
 
 
 # Imports
 import torch
-import torchvision
 import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
 import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
 import torch.nn.functional as F  # All functions that don't have any parameters
@@ -18,9 +20,10 @@ from torch.utils.data import (
 )  # Gives easier dataset managment and creates mini batches
 import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
 import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+from tqdm import tqdm  # progress bar
 
 # Set device
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = "cuda" if torch.cuda.is_available() else "cpu"
 
 # Hyperparameters
 input_size = 28
@@ -28,7 +31,7 @@ sequence_length = 28
 num_layers = 2
 hidden_size = 256
 num_classes = 10
-learning_rate = 0.001
+learning_rate = 3e-4
 batch_size = 64
 num_epochs = 2
 
@@ -47,7 +50,7 @@ class BRNN(nn.Module):
         h0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device)
         c0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device)
 
-        out, _ = self.lstm(x, (h0, c0))
+        out, _ = self.lstm(x)
         out = self.fc(out[:, -1, :])
 
         return out
@@ -74,7 +77,7 @@ optimizer = optim.Adam(model.parameters(), lr=learning_rate)
 
 # Train Network
 for epoch in range(num_epochs):
-    for batch_idx, (data, targets) in enumerate(train_loader):
+    for batch_idx, (data, targets) in enumerate(tqdm(train_loader)):
         # Get data to cuda if possible
         data = data.to(device=device).squeeze(1)
         targets = targets.to(device=device)
@@ -90,9 +93,8 @@ for epoch in range(num_epochs):
         # gradient descent or adam step
         optimizer.step()
 
+
 # Check accuracy on training & test to see how good our model
-
-
 def check_accuracy(loader, model):
     if loader.dataset.train:
         print("Checking accuracy on training data")

ML/Pytorch/Basics/pytorch_init_weights.py

@@ -1,12 +1,16 @@
 """
 Example code of how to initialize weights for a simple CNN network.
+Usually this is not needed as default initialization is usually good,
+but sometimes it can be useful to initialize weights in a specific way.
+This way of doing it should generalize to other network types just make 
+sure to specify and change the modules you wish to modify.
 
 Video explanation: https://youtu.be/xWQ-p_o0Uik
 Got any questions leave a comment on youtube :)
 
 Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
 *    2020-04-10 Initial coding
-
+*    2022-12-16 Updated with more detailed comments, and checked code still functions as intended.
 """
 
 # Imports
@@ -20,17 +24,17 @@ class CNN(nn.Module):
         self.conv1 = nn.Conv2d(
             in_channels=in_channels,
             out_channels=6,
-            kernel_size=(3, 3),
+            kernel_size=3,
-            stride=(1, 1),
+            stride=1,
-            padding=(1, 1),
+            padding=1,
         )
         self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
         self.conv2 = nn.Conv2d(
             in_channels=6,
             out_channels=16,
-            kernel_size=(3, 3),
+            kernel_size=3,
-            stride=(1, 1),
+            stride=1,
-            padding=(1, 1),
+            padding=1,
         )
         self.fc1 = nn.Linear(16 * 7 * 7, num_classes)
         self.initialize_weights()

ML/Pytorch/Basics/pytorch_loadsave.py

@@ -9,7 +9,8 @@ Video explanation of code & how to save and load model: https://youtu.be/g6kQl_E
 Got any questions leave a comment on youtube :)
 
 Coded by Aladdin Persson <aladdin dot person at hotmail dot com>
--   2020-04-07 Initial programming
+*   2020-04-07 Initial programming
+*   2022-12-16 Updated with more detailed comments, and checked code still functions as intended.
 
 """
 
@@ -39,7 +40,9 @@ def load_checkpoint(checkpoint, model, optimizer):
 
 def main():
     # Initialize network
-    model = torchvision.models.vgg16(pretrained=False)
+    model = torchvision.models.vgg16(
+        weights=None
+    )  # pretrained=False deprecated, use weights instead
     optimizer = optim.Adam(model.parameters())
 
     checkpoint = {"state_dict": model.state_dict(), "optimizer": optimizer.state_dict()}

ML/Pytorch/Basics/pytorch_rnn_gru_lstm.py

@@ -3,22 +3,24 @@ Example code of a simple RNN, GRU, LSTM on the MNIST dataset.
 
 Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
 *    2020-05-09 Initial coding
+*    2022-12-16 Updated with more detailed comments, docstrings to functions, and checked code still functions as intended.
 
 """
 
 # Imports
 import torch
-import torchvision  # torch package for vision related things
 import torch.nn.functional as F  # Parameterless functions, like (some) activation functions
 import torchvision.datasets as datasets  # Standard datasets
 import torchvision.transforms as transforms  # Transformations we can perform on our dataset for augmentation
 from torch import optim  # For optimizers like SGD, Adam, etc.
 from torch import nn  # All neural network modules
-from torch.utils.data import DataLoader  # Gives easier dataset managment by creating mini batches etc.
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment by creating mini batches etc.
 from tqdm import tqdm  # For a nice progress bar!
 
 # Set device
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = "cuda" if torch.cuda.is_available() else "cpu"
 
 # Hyperparameters
 input_size = 28
@@ -100,8 +102,12 @@ class RNN_LSTM(nn.Module):
 
 
 # Load Data
-train_dataset = datasets.MNIST(root="dataset/", train=True, transform=transforms.ToTensor(), download=True)
+train_dataset = datasets.MNIST(
-test_dataset = datasets.MNIST(root="dataset/", train=False, transform=transforms.ToTensor(), download=True)
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+test_dataset = datasets.MNIST(
+    root="dataset/", train=False, transform=transforms.ToTensor(), download=True
+)
 train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
 test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
 

ML/Pytorch/CNN_architectures/pytorch_efficientnet.py

@@ -173,10 +173,3 @@ def test():
     print(model(x).shape) # (num_examples, num_classes)
 
 test()
-
-
-
-
-
-
-