Initial commit

ML/Pytorch/Basics/albumentations_tutorial/classification.py

@@ -0,0 +1,31 @@
+import cv2
+import albumentations as A
+import numpy as np
+from utils import plot_examples
+from PIL import Image
+
+image = Image.open("images/elon.jpeg")
+
+transform = A.Compose(
+    [
+        A.Resize(width=1920, height=1080),
+        A.RandomCrop(width=1280, height=720),
+        A.Rotate(limit=40, p=0.9, border_mode=cv2.BORDER_CONSTANT),
+        A.HorizontalFlip(p=0.5),
+        A.VerticalFlip(p=0.1),
+        A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25, p=0.9),
+        A.OneOf([
+            A.Blur(blur_limit=3, p=0.5),
+            A.ColorJitter(p=0.5),
+        ], p=1.0),
+    ]
+)
+
+images_list = [image]
+image = np.array(image)
+for i in range(15):
+    augmentations = transform(image=image)
+    augmented_img = augmentations["image"]
+    images_list.append(augmented_img)
+plot_examples(images_list)
+

ML/Pytorch/Basics/albumentations_tutorial/detection.py

@@ -0,0 +1,41 @@
+import cv2
+import albumentations as A
+import numpy as np
+from utils import plot_examples
+from PIL import Image
+
+image = cv2.imread("images/cat.jpg")
+image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+bboxes = [[13, 170, 224, 410]]
+
+# Pascal_voc (x_min, y_min, x_max, y_max), YOLO, COCO
+
+transform = A.Compose(
+    [
+        A.Resize(width=1920, height=1080),
+        A.RandomCrop(width=1280, height=720),
+        A.Rotate(limit=40, p=0.9, border_mode=cv2.BORDER_CONSTANT),
+        A.HorizontalFlip(p=0.5),
+        A.VerticalFlip(p=0.1),
+        A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25, p=0.9),
+        A.OneOf([
+            A.Blur(blur_limit=3, p=0.5),
+            A.ColorJitter(p=0.5),
+        ], p=1.0),
+    ], bbox_params=A.BboxParams(format="pascal_voc", min_area=2048,
+                                min_visibility=0.3, label_fields=[])
+)
+
+images_list = [image]
+saved_bboxes = [bboxes[0]]
+for i in range(15):
+    augmentations = transform(image=image, bboxes=bboxes)
+    augmented_img = augmentations["image"]
+
+    if len(augmentations["bboxes"]) == 0:
+        continue
+
+    images_list.append(augmented_img)
+    saved_bboxes.append(augmentations["bboxes"][0])
+
+plot_examples(images_list, saved_bboxes)

ML/Pytorch/Basics/albumentations_tutorial/full_pytorch_example.py

@@ -0,0 +1,62 @@
+import torch
+import numpy as np
+import cv2
+from PIL import Image
+import torch.nn as nn
+import albumentations as A
+from albumentations.pytorch import ToTensorV2
+from torch.utils.data import Dataset
+import os
+
+class ImageFolder(Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ImageFolder, self).__init__()
+        self.data = []
+        self.root_dir = root_dir
+        self.transform = transform
+        self.class_names = os.listdir(root_dir)
+
+        for index, name in enumerate(self.class_names):
+            files = os.listdir(os.path.join(root_dir, name))
+            self.data += list(zip(files, [index]*len(files)))
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        img_file, label = self.data[index]
+        root_and_dir = os.path.join(self.root_dir, self.class_names[label])
+        image = np.array(Image.open(os.path.join(root_and_dir, img_file)))
+
+        if self.transform is not None:
+            augmentations = self.transform(image=image)
+            image = augmentations["image"]
+
+        return image, label
+
+
+transform = A.Compose(
+    [
+        A.Resize(width=1920, height=1080),
+        A.RandomCrop(width=1280, height=720),
+        A.Rotate(limit=40, p=0.9, border_mode=cv2.BORDER_CONSTANT),
+        A.HorizontalFlip(p=0.5),
+        A.VerticalFlip(p=0.1),
+        A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25, p=0.9),
+        A.OneOf([
+            A.Blur(blur_limit=3, p=0.5),
+            A.ColorJitter(p=0.5),
+        ], p=1.0),
+        A.Normalize(
+            mean=[0, 0, 0],
+            std=[1, 1, 1],
+            max_pixel_value=255,
+        ),
+        ToTensorV2(),
+    ]
+)
+
+dataset = ImageFolder(root_dir="cat_dogs", transform=transform)
+
+for x,y in dataset:
+    print(x.shape)

ML/Pytorch/Basics/albumentations_tutorial/segmentation.py

@@ -0,0 +1,37 @@
+import cv2
+import albumentations as A
+import numpy as np
+from utils import plot_examples
+from PIL import Image
+
+image = Image.open("images/elon.jpeg")
+mask = Image.open("images/mask.jpeg")
+mask2 = Image.open("images/second_mask.jpeg")
+
+transform = A.Compose(
+    [
+        A.Resize(width=1920, height=1080),
+        A.RandomCrop(width=1280, height=720),
+        A.Rotate(limit=40, p=0.9, border_mode=cv2.BORDER_CONSTANT),
+        A.HorizontalFlip(p=0.5),
+        A.VerticalFlip(p=0.1),
+        A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25, p=0.9),
+        A.OneOf([
+            A.Blur(blur_limit=3, p=0.5),
+            A.ColorJitter(p=0.5),
+        ], p=1.0),
+    ]
+)
+
+images_list = [image]
+image = np.array(image)
+mask = np.array(mask) # np.asarray(mask), np.array(mask)
+mask2 = np.array(mask2)
+for i in range(4):
+    augmentations = transform(image=image, masks=[mask, mask2])
+    augmented_img = augmentations["image"]
+    augmented_masks = augmentations["masks"]
+    images_list.append(augmented_img)
+    images_list.append(augmented_masks[0])
+    images_list.append(augmented_masks[1])
+plot_examples(images_list)

ML/Pytorch/Basics/albumentations_tutorial/utils.py

@@ -0,0 +1,36 @@
+import random
+import cv2
+from matplotlib import pyplot as plt
+import matplotlib.patches as patches
+import numpy as np
+import albumentations as A
+
+
+def visualize(image):
+    plt.figure(figsize=(10, 10))
+    plt.axis('off')
+    plt.imshow(image)
+    plt.show()
+
+
+def plot_examples(images, bboxes=None):
+    fig = plt.figure(figsize=(15, 15))
+    columns = 4
+    rows = 5
+
+    for i in range(1, len(images)):
+        if bboxes is not None:
+            img = visualize_bbox(images[i - 1], bboxes[i - 1], class_name="Elon")
+        else:
+            img = images[i-1]
+        fig.add_subplot(rows, columns, i)
+        plt.imshow(img)
+    plt.show()
+
+
+# From https://albumentations.ai/docs/examples/example_bboxes/
+def visualize_bbox(img, bbox, class_name, color=(255, 0, 0), thickness=5):
+    """Visualizes a single bounding box on the image"""
+    x_min, y_min, x_max, y_max = map(int, bbox)
+    cv2.rectangle(img, (x_min, y_min), (x_max, y_max), color, thickness)
+    return img

ML/Pytorch/Basics/custom_dataset/cats_dogs.csv

@@ -0,0 +1,11 @@
+Animal,Label
+cat.0.jpg,0
+cat.1.jpg,0
+cat.2.jpg,0
+cat.3.jpg,0
+cat.4.jpg,0
+cat.5.jpg,0
+cat.6.jpg,0
+cat.7.jpg,0
+dog.0.jpg,1
+dog.1.jpg,1

ML/Pytorch/Basics/custom_dataset/custom_FCNN.py

@@ -0,0 +1,131 @@
+# Imports
+import os
+from typing import Union
+
+import torch.nn.functional as F  # All functions that don't have any parameters
+import pandas as pd
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torchvision
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+from pandas import io
+
+# from skimage import io
+from torch.utils.data import (
+    Dataset,
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+
+
+# Create Fully Connected Network
+class NN(nn.Module):
+    def __init__(self, input_size, num_classes):
+        super(NN, self).__init__()
+        self.fc1 = nn.Linear(input_size, 50)
+        self.fc2 = nn.Linear(50, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+
+class SoloDataset(Dataset):
+    def __init__(self, csv_file, root_dir, transform=None):
+        self.annotations = pd.read_csv(csv_file)
+        self.root_dir = root_dir
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.annotations)
+
+    def __getitem__(self, index):
+        x_data = self.annotations.iloc[index, 0:11]
+        x_data = torch.tensor(x_data)
+        y_label = torch.tensor(int(self.annotations.iloc[index, 11]))
+
+        return (x_data.float(), y_label)
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+num_classes = 26
+learning_rate = 1e-3
+batch_size = 5
+num_epochs = 30
+input_size = 11
+
+# Load Data
+dataset = SoloDataset(
+    csv_file="power.csv", root_dir="test123", transform=transforms.ToTensor()
+)
+train_set, test_set = torch.utils.data.random_split(dataset, [2900, 57])
+train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)
+test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)
+
+# Model
+model = NN(input_size=input_size, num_classes=num_classes).to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+print(len(train_set))
+print(len(test_set))
+# Train Network
+for epoch in range(num_epochs):
+    losses = []
+
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        losses.append(loss.item())
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+    print(f"Cost at epoch {epoch} is {sum(losses) / len(losses)}")
+
+
+# Check accuracy on training to see how good our model is
+def check_accuracy(loader, model):
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct) / float(num_samples) * 100:.2f}"
+        )
+
+    model.train()
+
+
+print("Checking accuracy on Training Set")
+check_accuracy(train_loader, model)
+
+print("Checking accuracy on Test Set")
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/custom_dataset/custom_dataset.py

@@ -0,0 +1,130 @@
+"""
+Example of how to create custom dataset in Pytorch. In this case
+we have images of cats and dogs in a separate folder and a csv
+file containing the name to the jpg file as well as the target
+label (0 for cat, 1 for dog).
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-03 Initial coding
+
+"""
+
+# Imports
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+import torchvision
+import os
+import pandas as pd
+from skimage import io
+from torch.utils.data import (
+    Dataset,
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+
+
+class CatsAndDogsDataset(Dataset):
+    def __init__(self, csv_file, root_dir, transform=None):
+        self.annotations = pd.read_csv(csv_file)
+        self.root_dir = root_dir
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.annotations)
+
+    def __getitem__(self, index):
+        img_path = os.path.join(self.root_dir, self.annotations.iloc[index, 0])
+        image = io.imread(img_path)
+        y_label = torch.tensor(int(self.annotations.iloc[index, 1]))
+
+        if self.transform:
+            image = self.transform(image)
+
+        return (image, y_label)
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+in_channel = 3
+num_classes = 2
+learning_rate = 1e-3
+batch_size = 32
+num_epochs = 10
+
+# Load Data
+dataset = CatsAndDogsDataset(
+    csv_file="cats_dogs.csv",
+    root_dir="cats_dogs_resized",
+    transform=transforms.ToTensor(),
+)
+
+# Dataset is actually a lot larger ~25k images, just took out 10 pictures
+# to upload to Github. It's enough to understand the structure and scale
+# if you got more images.
+train_set, test_set = torch.utils.data.random_split(dataset, [5, 5])
+train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)
+test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)
+
+# Model
+model = torchvision.models.googlenet(pretrained=True)
+model.to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    losses = []
+
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        losses.append(loss.item())
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+    print(f"Cost at epoch {epoch} is {sum(losses)/len(losses)}")
+
+# Check accuracy on training to see how good our model is
+def check_accuracy(loader, model):
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+print("Checking accuracy on Training Set")
+check_accuracy(train_loader, model)
+
+print("Checking accuracy on Test Set")
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/custom_dataset_txt/loader_customtext.py

@@ -0,0 +1,142 @@
+import os  # when loading file paths
+import pandas as pd  # for lookup in annotation file
+import spacy  # for tokenizer
+import torch
+from torch.nn.utils.rnn import pad_sequence  # pad batch
+from torch.utils.data import DataLoader, Dataset
+from PIL import Image  # Load img
+import torchvision.transforms as transforms
+
+
+# We want to convert text -> numerical values
+# 1. We need a Vocabulary mapping each word to a index
+# 2. We need to setup a Pytorch dataset to load the data
+# 3. Setup padding of every batch (all examples should be
+#    of same seq_len and setup dataloader)
+# Note that loading the image is very easy compared to the text!
+
+# Download with: python -m spacy download en
+spacy_eng = spacy.load("en")
+
+
+class Vocabulary:
+    def __init__(self, freq_threshold):
+        self.itos = {0: "<PAD>", 1: "<SOS>", 2: "<EOS>", 3: "<UNK>"}
+        self.stoi = {"<PAD>": 0, "<SOS>": 1, "<EOS>": 2, "<UNK>": 3}
+        self.freq_threshold = freq_threshold
+
+    def __len__(self):
+        return len(self.itos)
+
+    @staticmethod
+    def tokenizer_eng(text):
+        return [tok.text.lower() for tok in spacy_eng.tokenizer(text)]
+
+    def build_vocabulary(self, sentence_list):
+        frequencies = {}
+        idx = 4
+
+        for sentence in sentence_list:
+            for word in self.tokenizer_eng(sentence):
+                if word not in frequencies:
+                    frequencies[word] = 1
+
+                else:
+                    frequencies[word] += 1
+
+                if frequencies[word] == self.freq_threshold:
+                    self.stoi[word] = idx
+                    self.itos[idx] = word
+                    idx += 1
+
+    def numericalize(self, text):
+        tokenized_text = self.tokenizer_eng(text)
+
+        return [
+            self.stoi[token] if token in self.stoi else self.stoi["<UNK>"]
+            for token in tokenized_text
+        ]
+
+
+class FlickrDataset(Dataset):
+    def __init__(self, root_dir, captions_file, transform=None, freq_threshold=5):
+        self.root_dir = root_dir
+        self.df = pd.read_csv(captions_file)
+        self.transform = transform
+
+        # Get img, caption columns
+        self.imgs = self.df["image"]
+        self.captions = self.df["caption"]
+
+        # Initialize vocabulary and build vocab
+        self.vocab = Vocabulary(freq_threshold)
+        self.vocab.build_vocabulary(self.captions.tolist())
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, index):
+        caption = self.captions[index]
+        img_id = self.imgs[index]
+        img = Image.open(os.path.join(self.root_dir, img_id)).convert("RGB")
+
+        if self.transform is not None:
+            img = self.transform(img)
+
+        numericalized_caption = [self.vocab.stoi["<SOS>"]]
+        numericalized_caption += self.vocab.numericalize(caption)
+        numericalized_caption.append(self.vocab.stoi["<EOS>"])
+
+        return img, torch.tensor(numericalized_caption)
+
+
+class MyCollate:
+    def __init__(self, pad_idx):
+        self.pad_idx = pad_idx
+
+    def __call__(self, batch):
+        imgs = [item[0].unsqueeze(0) for item in batch]
+        imgs = torch.cat(imgs, dim=0)
+        targets = [item[1] for item in batch]
+        targets = pad_sequence(targets, batch_first=False, padding_value=self.pad_idx)
+
+        return imgs, targets
+
+
+def get_loader(
+    root_folder,
+    annotation_file,
+    transform,
+    batch_size=32,
+    num_workers=8,
+    shuffle=True,
+    pin_memory=True,
+):
+    dataset = FlickrDataset(root_folder, annotation_file, transform=transform)
+
+    pad_idx = dataset.vocab.stoi["<PAD>"]
+
+    loader = DataLoader(
+        dataset=dataset,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        shuffle=shuffle,
+        pin_memory=pin_memory,
+        collate_fn=MyCollate(pad_idx=pad_idx),
+    )
+
+    return loader, dataset
+
+
+if __name__ == "__main__":
+    transform = transforms.Compose(
+        [transforms.Resize((224, 224)), transforms.ToTensor(),]
+    )
+
+    loader, dataset = get_loader(
+        "flickr8k/images/", "flickr8k/captions.txt", transform=transform
+    )
+
+    for idx, (imgs, captions) in enumerate(loader):
+        print(imgs.shape)
+        print(captions.shape)

ML/Pytorch/Basics/pytorch_bidirectional_lstm.py

@@ -0,0 +1,125 @@
+"""
+Example code of a simple bidirectional LSTM on the MNIST dataset.
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-05-09 Initial coding
+
+"""
+
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+input_size = 28
+sequence_length = 28
+num_layers = 2
+hidden_size = 256
+num_classes = 10
+learning_rate = 0.001
+batch_size = 64
+num_epochs = 2
+
+# Create a bidirectional LSTM
+class BRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, num_classes):
+        super(BRNN, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(
+            input_size, hidden_size, num_layers, batch_first=True, bidirectional=True
+        )
+        self.fc = nn.Linear(hidden_size * 2, num_classes)
+
+    def forward(self, x):
+        h0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device)
+        c0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device)
+
+        out, _ = self.lstm(x, (h0, c0))
+        out = self.fc(out[:, -1, :])
+
+        return out
+
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+
+test_dataset = datasets.MNIST(
+    root="dataset/", train=False, transform=transforms.ToTensor(), download=True
+)
+
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
+
+# Initialize network
+model = BRNN(input_size, hidden_size, num_layers, num_classes).to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device).squeeze(1)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+# Check accuracy on training & test to see how good our model
+
+
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device).squeeze(1)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy  \
+              {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/pytorch_init_weights.py

@@ -0,0 +1,69 @@
+"""
+Example code of how to initialize weights for a simple CNN network.
+
+Video explanation: https://youtu.be/xWQ-p_o0Uik
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-10 Initial coding
+
+"""
+
+# Imports
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.nn.functional as F  # All functions that don't have any parameters
+
+
+class CNN(nn.Module):
+    def __init__(self, in_channels, num_classes):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(
+            in_channels=in_channels,
+            out_channels=6,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
+        self.conv2 = nn.Conv2d(
+            in_channels=6,
+            out_channels=16,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.fc1 = nn.Linear(16 * 7 * 7, num_classes)
+        self.initialize_weights()
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.pool(x)
+        x = F.relu(self.conv2(x))
+        x = self.pool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+
+        return x
+
+    def initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_uniform_(m.weight)
+
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.Linear):
+                nn.init.kaiming_uniform_(m.weight)
+                nn.init.constant_(m.bias, 0)
+
+
+if __name__ == "__main__":
+    model = CNN(in_channels=3, num_classes=10)
+
+    for param in model.parameters():
+        print(param)

ML/Pytorch/Basics/pytorch_loadsave.py

@@ -0,0 +1,54 @@
+"""
+Small code example of how to save and load checkpoint of a model.
+This example doesn't perform any training, so it would be quite useless.
+
+In practice you would save the model as you train, and then load before 
+continuining training at another point.
+
+Video explanation of code & how to save and load model: https://youtu.be/g6kQl_EFn84
+Got any questions leave a comment on youtube :)
+
+Coded by Aladdin Persson <aladdin dot person at hotmail dot com>
+-   2020-04-07 Initial programming
+
+"""
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+
+def save_checkpoint(state, filename="my_checkpoint.pth.tar"):
+    print("=> Saving checkpoint")
+    torch.save(state, filename)
+
+
+def load_checkpoint(checkpoint, model, optimizer):
+    print("=> Loading checkpoint")
+    model.load_state_dict(checkpoint["state_dict"])
+    optimizer.load_state_dict(checkpoint["optimizer"])
+
+
+def main():
+    # Initialize network
+    model = torchvision.models.vgg16(pretrained=False)
+    optimizer = optim.Adam(model.parameters())
+
+    checkpoint = {"state_dict": model.state_dict(), "optimizer": optimizer.state_dict()}
+    # Try save checkpoint
+    save_checkpoint(checkpoint)
+
+    # Try load checkpoint
+    load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer)
+
+
+if __name__ == "__main__":
+    main()

ML/Pytorch/Basics/pytorch_lr_ratescheduler.py

@@ -0,0 +1,107 @@
+"""
+Example code of how to use a learning rate scheduler simple, in this
+case with a (very) small and simple Feedforward Network training on MNIST
+dataset with a learning rate scheduler. In this case ReduceLROnPlateau
+scheduler is used, but can easily be changed to any of the other schedulers
+available.
+
+Video explanation: https://youtu.be/P31hB37g4Ak
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-10 Initial programming
+
+"""
+
+# Imports
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+num_classes = 10
+learning_rate = 0.1
+batch_size = 128
+num_epochs = 100
+
+# Define a very simple model
+model = nn.Sequential(nn.Linear(784, 50), nn.ReLU(), nn.Linear(50, 10)).to(device)
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Define Scheduler
+scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+    optimizer, factor=0.1, patience=5, verbose=True
+)
+
+# Train Network
+for epoch in range(1, num_epochs):
+    losses = []
+
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.reshape(data.shape[0], -1)
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        losses.append(loss.item())
+
+        # backward
+        loss.backward()
+
+        # gradient descent or adam step
+        # scheduler.step(loss)
+        optimizer.step()
+        optimizer.zero_grad()
+
+    mean_loss = sum(losses) / len(losses)
+
+    # After each epoch do scheduler.step, note in this scheduler we need to send
+    # in loss for that epoch!
+    scheduler.step(mean_loss)
+    print(f"Cost at epoch {epoch} is {mean_loss}")
+
+# Check accuracy on training & test to see how good our model
+def check_accuracy(loader, model):
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)

ML/Pytorch/Basics/pytorch_mixed_precision_example.py

@@ -0,0 +1,99 @@
+# Imports
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import DataLoader  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+
+# Simple CNN
+class CNN(nn.Module):
+    def __init__(self, in_channels=1, num_classes=10):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(in_channels=1, out_channels=420, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
+        self.conv2 = nn.Conv2d(in_channels=420, out_channels=1000, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.fc1 = nn.Linear(1000 * 7 * 7, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.pool(x)
+        x = F.relu(self.conv2(x))
+        x = self.pool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+
+        return x
+
+
+# Set device
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# Hyperparameters
+in_channel = 1
+num_classes = 10
+learning_rate = 0.001
+batch_size = 100
+num_epochs = 5
+
+# Load Data
+train_dataset = datasets.MNIST(root='dataset/', train=True, transform=transforms.ToTensor(), download=True)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+test_dataset = datasets.MNIST(root='dataset/', train=False, transform=transforms.ToTensor(), download=True)
+test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
+
+# Initialize network
+model = CNN().to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Necessary for FP16
+scaler = torch.cuda.amp.GradScaler()
+
+# Train Network
+for epoch in range(num_epochs):
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        with torch.cuda.amp.autocast():
+            scores = model(data)
+            loss = criterion(scores, targets)
+
+        # backward
+        optimizer.zero_grad()
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+
+
+# Check accuracy on training & test to see how good our model
+
+def check_accuracy(loader, model):
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(f'Got {num_correct} / {num_samples} with accuracy {float(num_correct) / float(num_samples) * 100:.2f}')
+
+    model.train()
+
+
+check_accuracy(train_loader, model)
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/pytorch_pretrain_finetune.py

@@ -0,0 +1,123 @@
+"""
+Shows a small example of how to load a pretrain model (VGG16) from PyTorch,
+and modifies this to train on the CIFAR10 dataset. The same method generalizes
+well to other datasets, but the modifications to the network may need to be changed.
+
+Video explanation: https://youtu.be/U4bHxEhMGNk
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-08 Initial coding
+
+"""
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+num_classes = 10
+learning_rate = 1e-3
+batch_size = 1024
+num_epochs = 5
+
+# Simple Identity class that let's input pass without changes
+class Identity(nn.Module):
+    def __init__(self):
+        super(Identity, self).__init__()
+
+    def forward(self, x):
+        return x
+
+
+# Load pretrain model & modify it
+model = torchvision.models.vgg16(pretrained=True)
+
+# If you want to do finetuning then set requires_grad = False
+# Remove these two lines if you want to train entire model,
+# and only want to load the pretrain weights.
+for param in model.parameters():
+    param.requires_grad = False
+
+model.avgpool = Identity()
+model.classifier = nn.Sequential(
+    nn.Linear(512, 100), nn.ReLU(), nn.Linear(100, num_classes)
+)
+model.to(device)
+
+
+# Load Data
+train_dataset = datasets.CIFAR10(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    losses = []
+
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        losses.append(loss.item())
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+    print(f"Cost at epoch {epoch} is {sum(losses)/len(losses):.5f}")
+
+# Check accuracy on training & test to see how good our model
+
+
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)

ML/Pytorch/Basics/pytorch_progress_bar.py

@@ -0,0 +1,41 @@
+import torch
+import torch.nn as nn
+from tqdm import tqdm
+from torch.utils.data import TensorDataset, DataLoader
+
+# Create a simple toy dataset example, normally this
+# would be doing custom class with __getitem__ etc,
+# which we have done in custom dataset tutorials
+x = torch.randn((1000, 3, 224, 224))
+y = torch.randint(low=0, high=10, size=(1000, 1))
+ds = TensorDataset(x, y)
+loader = DataLoader(ds, batch_size=8)
+
+
+model = nn.Sequential(
+    nn.Conv2d(3, 10, kernel_size=3, padding=1, stride=1),
+    nn.Flatten(),
+    nn.Linear(10*224*224, 10),
+)
+
+NUM_EPOCHS = 100
+for epoch in range(NUM_EPOCHS):
+    loop = tqdm(loader)
+    for idx, (x, y) in enumerate(loop):
+        scores = model(x)
+
+        # here we would compute loss, backward, optimizer step etc.
+        # you know how it goes, but now you have a nice progress bar
+        # with tqdm
+
+        # then at the bottom if you want additional info shown, you can
+        # add it here, for loss and accuracy you would obviously compute
+        # but now we just set them to random values
+        loop.set_description(f"Epoch [{epoch}/{NUM_EPOCHS}]")
+        loop.set_postfix(loss=torch.rand(1).item(), acc=torch.rand(1).item())
+
+# There you go. Hope it was useful :)
+
+
+
+

ML/Pytorch/Basics/pytorch_rnn_gru_lstm.py

@@ -0,0 +1,172 @@
+"""
+Example code of a simple RNN, GRU, LSTM on the MNIST dataset.
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-05-09 Initial coding
+
+"""
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+input_size = 28
+hidden_size = 256
+num_layers = 2
+num_classes = 10
+sequence_length = 28
+learning_rate = 0.005
+batch_size = 64
+num_epochs = 2
+
+# Recurrent neural network (many-to-one)
+class RNN(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, num_classes):
+        super(RNN, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size * sequence_length, num_classes)
+
+    def forward(self, x):
+        # Set initial hidden and cell states
+        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+
+        # Forward propagate LSTM
+        out, _ = self.rnn(x, h0)
+        out = out.reshape(out.shape[0], -1)
+
+        # Decode the hidden state of the last time step
+        out = self.fc(out)
+        return out
+
+
+# Recurrent neural network with GRU (many-to-one)
+class RNN_GRU(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, num_classes):
+        super(RNN_GRU, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.gru = nn.GRU(input_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size * sequence_length, num_classes)
+
+    def forward(self, x):
+        # Set initial hidden and cell states
+        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+
+        # Forward propagate LSTM
+        out, _ = self.gru(x, h0)
+        out = out.reshape(out.shape[0], -1)
+
+        # Decode the hidden state of the last time step
+        out = self.fc(out)
+        return out
+
+
+# Recurrent neural network with LSTM (many-to-one)
+class RNN_LSTM(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, num_classes):
+        super(RNN_LSTM, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size * sequence_length, num_classes)
+
+    def forward(self, x):
+        # Set initial hidden and cell states
+        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+
+        # Forward propagate LSTM
+        out, _ = self.lstm(
+            x, (h0, c0)
+        )  # out: tensor of shape (batch_size, seq_length, hidden_size)
+        out = out.reshape(out.shape[0], -1)
+
+        # Decode the hidden state of the last time step
+        out = self.fc(out)
+        return out
+
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+
+test_dataset = datasets.MNIST(
+    root="dataset/", train=False, transform=transforms.ToTensor(), download=True
+)
+
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
+
+# Initialize network
+model = RNN_LSTM(input_size, hidden_size, num_layers, num_classes).to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device).squeeze(1)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+# Check accuracy on training & test to see how good our model
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+
+    # Set model to eval
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device).squeeze(1)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with \
+              accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+    # Set model back to train
+    model.train()
+
+
+check_accuracy(train_loader, model)
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/pytorch_simple_CNN.py

@@ -0,0 +1,134 @@
+"""
+Example code of a simple CNN network training on MNIST dataset.
+The code is intended to show how to create a CNN network as well
+as how to initialize loss, optimizer, etc. in a simple way to get
+training to work with function that checks accuracy as well.
+
+Video explanation: https://youtu.be/wnK3uWv_WkU
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-08 Initial coding
+
+"""
+
+# Imports
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Simple CNN
+class CNN(nn.Module):
+    def __init__(self, in_channels=1, num_classes=10):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(
+            in_channels=1,
+            out_channels=8,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
+        self.conv2 = nn.Conv2d(
+            in_channels=8,
+            out_channels=16,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.fc1 = nn.Linear(16 * 7 * 7, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.pool(x)
+        x = F.relu(self.conv2(x))
+        x = self.pool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+
+        return x
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+in_channel = 1
+num_classes = 10
+learning_rate = 0.001
+batch_size = 64
+num_epochs = 5
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+test_dataset = datasets.MNIST(
+    root="dataset/", train=False, transform=transforms.ToTensor(), download=True
+)
+test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
+
+# Initialize network
+model = CNN().to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+# Check accuracy on training & test to see how good our model
+
+
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/pytorch_simple_fullynet.py

@@ -0,0 +1,120 @@
+"""
+Working code of a simple Fully Connected (FC) network training on MNIST dataset.
+The code is intended to show how to create a FC network as well
+as how to initialize loss, optimizer, etc. in a simple way to get
+training to work with function that checks accuracy as well.
+
+Video explanation: https://youtu.be/Jy4wM2X21u0
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-08 Initial coding
+
+"""
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Create Fully Connected Network
+class NN(nn.Module):
+    def __init__(self, input_size, num_classes):
+        super(NN, self).__init__()
+        self.fc1 = nn.Linear(input_size, 50)
+        self.fc2 = nn.Linear(50, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+input_size = 784
+num_classes = 10
+learning_rate = 0.001
+batch_size = 64
+num_epochs = 1
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+test_dataset = datasets.MNIST(
+    root="dataset/", train=False, transform=transforms.ToTensor(), download=True
+)
+test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
+
+# Initialize network
+model = NN(input_size=input_size, num_classes=num_classes).to(device)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # Get to correct shape
+        data = data.reshape(data.shape[0], -1)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+# Check accuracy on training & test to see how good our model
+
+
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+            x = x.reshape(x.shape[0], -1)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)
+check_accuracy(test_loader, model)

ML/Pytorch/Basics/pytorch_std_mean.py

@@ -0,0 +1,28 @@
+import torch
+import torchvision.transforms as transforms
+from torch.utils.data import DataLoader
+import torchvision.datasets as datasets
+from tqdm import tqdm
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+train_set = datasets.CIFAR10(root="ds/", transform=transforms.ToTensor(), download=True)
+train_loader = DataLoader(dataset=train_set, batch_size=64, shuffle=True)
+
+def get_mean_std(loader):
+    # var[X] = E[X**2] - E[X]**2
+    channels_sum, channels_sqrd_sum, num_batches = 0, 0, 0
+
+    for data, _ in tqdm(loader):
+        channels_sum += torch.mean(data, dim=[0, 2, 3])
+        channels_sqrd_sum += torch.mean(data ** 2, dim=[0, 2, 3])
+        num_batches += 1
+
+    mean = channels_sum / num_batches
+    std = (channels_sqrd_sum / num_batches - mean ** 2) ** 0.5
+
+    return mean, std
+
+
+mean, std = get_mean_std(train_loader)
+print(mean)
+print(std)

ML/Pytorch/Basics/pytorch_tensorbasics.py

@@ -0,0 +1,299 @@
+"""
+Walk through of a lot of different useful Tensor Operations, where we
+go through what I think are four main parts in:
+
+1. Initialization of a Tensor
+2. Tensor Mathematical Operations and Comparison
+3. Tensor Indexing
+4. Tensor Reshaping
+
+But also other things such as setting the device (GPU/CPU) and converting
+between different types (int, float etc) and how to convert a tensor to an
+numpy array and vice-versa.
+
+"""
+
+import torch
+
+# ================================================================= #
+#                        Initializing Tensor                        #
+# ================================================================= #
+
+device = "cuda" if torch.cuda.is_available() else "cpu"  # Cuda to run on GPU!
+
+# Initializing a Tensor in this case of shape 2x3 (2 rows, 3 columns)
+my_tensor = torch.tensor(
+    [[1, 2, 3], [4, 5, 6]], dtype=torch.float32, device=device, requires_grad=True
+)
+
+# A few tensor attributes
+print(
+    f"Information about tensor: {my_tensor}"
+)  # Prints data of the tensor, device and grad info
+print(
+    "Type of Tensor {my_tensor.dtype}"
+)  # Prints dtype of the tensor (torch.float32, etc)
+print(
+    f"Device Tensor is on {my_tensor.device}"
+)  # Prints cpu/cuda (followed by gpu number)
+print(f"Shape of tensor {my_tensor.shape}")  # Prints shape, in this case 2x3
+print(f"Requires gradient: {my_tensor.requires_grad}")  # Prints true/false
+
+# Other common initialization methods (there exists a ton more)
+x = torch.empty(size=(3, 3))  # Tensor of shape 3x3 with uninitialized data
+x = torch.zeros((3, 3))  # Tensor of shape 3x3 with values of 0
+x = torch.rand(
+    (3, 3)
+)  # Tensor of shape 3x3 with values from uniform distribution in interval [0,1)
+x = torch.ones((3, 3))  # Tensor of shape 3x3 with values of 1
+x = torch.eye(5, 5)  # Returns Identity Matrix I, (I <-> Eye), matrix of shape 2x3
+x = torch.arange(
+    start=0, end=5, step=1
+)  # Tensor [0, 1, 2, 3, 4], note, can also do: torch.arange(11)
+x = torch.linspace(start=0.1, end=1, steps=10)  # x = [0.1, 0.2, ..., 1]
+x = torch.empty(size=(1, 5)).normal_(
+    mean=0, std=1
+)  # Normally distributed with mean=0, std=1
+x = torch.empty(size=(1, 5)).uniform_(
+    0, 1
+)  # Values from a uniform distribution low=0, high=1
+x = torch.diag(torch.ones(3))  # Diagonal matrix of shape 3x3
+
+# How to make initialized tensors to other types (int, float, double)
+# These will work even if you're on CPU or CUDA!
+tensor = torch.arange(4)  # [0, 1, 2, 3] Initialized as int64 by default
+print(f"Converted Boolean: {tensor.bool()}")  # Converted to Boolean: 1 if nonzero
+print(f"Converted int16 {tensor.short()}")  # Converted to int16
+print(
+    f"Converted int64 {tensor.long()}"
+)  # Converted to int64 (This one is very important, used super often)
+print(f"Converted float16 {tensor.half()}")  # Converted to float16
+print(
+    f"Converted float32 {tensor.float()}"
+)  # Converted to float32 (This one is very important, used super often)
+print(f"Converted float64 {tensor.double()}")  # Converted to float64
+
+# Array to Tensor conversion and vice-versa
+import numpy as np
+
+np_array = np.zeros((5, 5))
+tensor = torch.from_numpy(np_array)
+np_array_again = (
+    tensor.numpy()
+)  # np_array_again will be same as np_array (perhaps with numerical round offs)
+
+# =============================================================================== #
+#                        Tensor Math & Comparison Operations                      #
+# =============================================================================== #
+
+x = torch.tensor([1, 2, 3])
+y = torch.tensor([9, 8, 7])
+
+# -- Addition --
+z1 = torch.empty(3)
+torch.add(x, y, out=z1)  # This is one way
+z2 = torch.add(x, y)  # This is another way
+z = x + y  # This is my preferred way, simple and clean.
+
+# -- Subtraction --
+z = x - y  # We can do similarly as the preferred way of addition
+
+# -- Division (A bit clunky) --
+z = torch.true_divide(x, y)  # Will do element wise division if of equal shape
+
+# -- Inplace Operations --
+t = torch.zeros(3)
+
+t.add_(x)  # Whenever we have operation followed by _ it will mutate the tensor in place
+t += x  # Also inplace: t = t + x is not inplace, bit confusing.
+
+# -- Exponentiation (Element wise if vector or matrices) --
+z = x.pow(2)  # z = [1, 4, 9]
+z = x ** 2  # z = [1, 4, 9]
+
+
+# -- Simple Comparison --
+z = x > 0  # Returns [True, True, True]
+z = x < 0  # Returns [False, False, False]
+
+# -- Matrix Multiplication --
+x1 = torch.rand((2, 5))
+x2 = torch.rand((5, 3))
+x3 = torch.mm(x1, x2)  # Matrix multiplication of x1 and x2, out shape: 2x3
+x3 = x1.mm(x2)  # Similar as line above
+
+# -- Matrix Exponentiation --
+matrix_exp = torch.rand(5, 5)
+print(
+    matrix_exp.matrix_power(3)
+)  # is same as matrix_exp (mm) matrix_exp (mm) matrix_exp
+
+# -- Element wise Multiplication --
+z = x * y  # z = [9, 16, 21] = [1*9, 2*8, 3*7]
+
+# -- Dot product --
+z = torch.dot(x, y)  # Dot product, in this case z = 1*9 + 2*8 + 3*7
+
+# -- Batch Matrix Multiplication --
+batch = 32
+n = 10
+m = 20
+p = 30
+tensor1 = torch.rand((batch, n, m))
+tensor2 = torch.rand((batch, m, p))
+out_bmm = torch.bmm(tensor1, tensor2)  # Will be shape: (b x n x p)
+
+# -- Example of broadcasting --
+x1 = torch.rand((5, 5))
+x2 = torch.ones((1, 5))
+z = (
+    x1 - x2
+)  # Shape of z is 5x5: How? The 1x5 vector (x2) is subtracted for each row in the 5x5 (x1)
+z = (
+    x1 ** x2
+)  # Shape of z is 5x5: How? Broadcasting! Element wise exponentiation for every row
+
+# Other useful tensor operations
+sum_x = torch.sum(
+    x, dim=0
+)  # Sum of x across dim=0 (which is the only dim in our case), sum_x = 6
+values, indices = torch.max(x, dim=0)  # Can also do x.max(dim=0)
+values, indices = torch.min(x, dim=0)  # Can also do x.min(dim=0)
+abs_x = torch.abs(x)  # Returns x where abs function has been applied to every element
+z = torch.argmax(x, dim=0)  # Gets index of the maximum value
+z = torch.argmin(x, dim=0)  # Gets index of the minimum value
+mean_x = torch.mean(x.float(), dim=0)  # mean requires x to be float
+z = torch.eq(x, y)  # Element wise comparison, in this case z = [False, False, False]
+sorted_y, indices = torch.sort(y, dim=0, descending=False)
+
+z = torch.clamp(x, min=0)
+# All values < 0 set to 0 and values > 0 unchanged (this is exactly ReLU function)
+# If you want to values over max_val to be clamped, do torch.clamp(x, min=min_val, max=max_val)
+
+x = torch.tensor([1, 0, 1, 1, 1], dtype=torch.bool)  # True/False values
+z = torch.any(x)  # will return True, can also do x.any() instead of torch.any(x)
+z = torch.all(
+    x
+)  # will return False (since not all are True), can also do x.all() instead of torch.all()
+
+# ============================================================= #
+#                        Tensor Indexing                        #
+# ============================================================= #
+
+batch_size = 10
+features = 25
+x = torch.rand((batch_size, features))
+
+# Get first examples features
+print(x[0].shape)  # shape [25], this is same as doing x[0,:]
+
+# Get the first feature for all examples
+print(x[:, 0].shape)  # shape [10]
+
+# For example: Want to access third example in the batch and the first ten features
+print(x[2, 0:10].shape)  # shape: [10]
+
+# For example we can use this to, assign certain elements
+x[0, 0] = 100
+
+# Fancy Indexing
+x = torch.arange(10)
+indices = [2, 5, 8]
+print(x[indices])  # x[indices] = [2, 5, 8]
+
+x = torch.rand((3, 5))
+rows = torch.tensor([1, 0])
+cols = torch.tensor([4, 0])
+print(x[rows, cols])  # Gets second row fifth column and first row first column
+
+# More advanced indexing
+x = torch.arange(10)
+print(x[(x < 2) | (x > 8)])  # will be [0, 1, 9]
+print(x[x.remainder(2) == 0])  # will be [0, 2, 4, 6, 8]
+
+# Useful operations for indexing
+print(
+    torch.where(x > 5, x, x * 2)
+)  # gives [0, 2, 4, 6, 8, 10, 6, 7, 8, 9], all values x > 5 yield x, else x*2
+x = torch.tensor([0, 0, 1, 2, 2, 3, 4]).unique()  # x = [0, 1, 2, 3, 4]
+print(
+    x.ndimension()
+)  # The number of dimensions, in this case 1. if x.shape is 5x5x5 ndim would be 3
+x = torch.arange(10)
+print(
+    x.numel()
+)  # The number of elements in x (in this case it's trivial because it's just a vector)
+
+# ============================================================= #
+#                        Tensor Reshaping                       #
+# ============================================================= #
+
+x = torch.arange(9)
+
+# Let's say we want to reshape it to be 3x3
+x_3x3 = x.view(3, 3)
+
+# We can also do (view and reshape are very similar)
+# and the differences are in simple terms (I'm no expert at this),
+# is that view acts on contiguous tensors meaning if the
+# tensor is stored contiguously in memory or not, whereas
+# for reshape it doesn't matter because it will copy the
+# tensor to make it contiguously stored, which might come
+# with some performance loss.
+x_3x3 = x.reshape(3, 3)
+
+# If we for example do:
+y = x_3x3.t()
+print(
+    y.is_contiguous()
+)  # This will return False and if we try to use view now, it won't work!
+# y.view(9) would cause an error, reshape however won't
+
+# This is because in memory it was stored [0, 1, 2, ... 8], whereas now it's [0, 3, 6, 1, 4, 7, 2, 5, 8]
+# The jump is no longer 1 in memory for one element jump (matrices are stored as a contiguous block, and
+# using pointers to construct these matrices). This is a bit complicated and I need to explore this more
+# as well, at least you know it's a problem to be cautious of! A solution is to do the following
+print(y.contiguous().view(9))  # Calling .contiguous() before view and it works
+
+# Moving on to another operation, let's say we want to add two tensors dimensions togethor
+x1 = torch.rand(2, 5)
+x2 = torch.rand(2, 5)
+print(torch.cat((x1, x2), dim=0).shape)  # Shape: 4x5
+print(torch.cat((x1, x2), dim=1).shape)  # Shape 2x10
+
+# Let's say we want to unroll x1 into one long vector with 10 elements, we can do:
+z = x1.view(-1)  # And -1 will unroll everything
+
+# If we instead have an additional dimension and we wish to keep those as is we can do:
+batch = 64
+x = torch.rand((batch, 2, 5))
+z = x.view(
+    batch, -1
+)  # And z.shape would be 64x10, this is very useful stuff and is used all the time
+
+# Let's say we want to switch x axis so that instead of 64x2x5 we have 64x5x2
+# I.e we want dimension 0 to stay, dimension 1 to become dimension 2, dimension 2 to become dimension 1
+# Basically you tell permute where you want the new dimensions to be, torch.transpose is a special case
+# of permute (why?)
+z = x.permute(0, 2, 1)
+
+# Splits x last dimension into chunks of 2 (since 5 is not integer div by 2) the last dimension
+# will be smaller, so it will split it into two tensors: 64x2x3 and 64x2x2
+z = torch.chunk(x, chunks=2, dim=1)
+print(z[0].shape)
+print(z[1].shape)
+
+# Let's say we want to add an additional dimension
+x = torch.arange(
+    10
+)  # Shape is [10], let's say we want to add an additional so we have 1x10
+print(x.unsqueeze(0).shape)  # 1x10
+print(x.unsqueeze(1).shape)  # 10x1
+
+# Let's say we have x which is 1x1x10 and we want to remove a dim so we have 1x10
+x = torch.arange(10).unsqueeze(0).unsqueeze(1)
+
+# Perhaps unsurprisingly
+z = x.squeeze(1)  # can also do .squeeze(0) both returns 1x10
+
+# That was some essential Tensor operations, hopefully you found it useful!

ML/Pytorch/Basics/pytorch_tensorboard_.py

@@ -0,0 +1,142 @@
+"""
+Example code of how to use the TensorBoard in PyTorch.
+This code uses a lot of different functions from TensorBoard
+and tries to have them all in a compact way, it might not be
+super clear exactly what calls does what, for that I recommend
+watching the YouTube video.
+
+Video explanation: https://youtu.be/RLqsxWaQdHE
+Got any questions leave a comment on youtube :)
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-17 Initial coding
+"""
+
+# Imports
+import torch
+import torchvision
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+from torch.utils.tensorboard import SummaryWriter  # to print to tensorboard
+
+# Simple CNN
+class CNN(nn.Module):
+    def __init__(self, in_channels=1, num_classes=10):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(
+            in_channels=in_channels, out_channels=8, kernel_size=3, stride=1, padding=1
+        )
+        self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
+        self.conv2 = nn.Conv2d(
+            in_channels=8, out_channels=16, kernel_size=3, stride=1, padding=1
+        )
+        self.fc1 = nn.Linear(16 * 7 * 7, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.pool(x)
+        x = F.relu(self.conv2(x))
+        x = self.pool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+        return x
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+in_channels = 1
+num_classes = 10
+num_epochs = 1
+
+# Load Data
+train_dataset = datasets.MNIST(
+    root="dataset/", train=True, transform=transforms.ToTensor(), download=True
+)
+
+# To do hyperparameter search, include more batch_sizes you want to try
+# and more learning rates!
+batch_sizes = [256]
+learning_rates = [0.001]
+classes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
+
+for batch_size in batch_sizes:
+    for learning_rate in learning_rates:
+        step = 0
+        # Initialize network
+        model = CNN(in_channels=in_channels, num_classes=num_classes)
+        model.to(device)
+        model.train()
+        criterion = nn.CrossEntropyLoss()
+        train_loader = DataLoader(
+            dataset=train_dataset, batch_size=batch_size, shuffle=True
+        )
+        optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0)
+        writer = SummaryWriter(
+            f"runs/MNIST/MiniBatchSize {batch_size} LR {learning_rate}"
+        )
+
+        # Visualize model in TensorBoard
+        images, _ = next(iter(train_loader))
+        writer.add_graph(model, images.to(device))
+        writer.close()
+
+        for epoch in range(num_epochs):
+            losses = []
+            accuracies = []
+
+            for batch_idx, (data, targets) in enumerate(train_loader):
+                # Get data to cuda if possible
+                data = data.to(device=device)
+                targets = targets.to(device=device)
+
+                # forward
+                scores = model(data)
+                loss = criterion(scores, targets)
+                losses.append(loss.item())
+
+                # backward
+                optimizer.zero_grad()
+                loss.backward()
+                optimizer.step()
+
+                # Calculate 'running' training accuracy
+                features = data.reshape(data.shape[0], -1)
+                img_grid = torchvision.utils.make_grid(data)
+                _, predictions = scores.max(1)
+                num_correct = (predictions == targets).sum()
+                running_train_acc = float(num_correct) / float(data.shape[0])
+                accuracies.append(running_train_acc)
+
+                # Plot things to tensorboard
+                class_labels = [classes[label] for label in predictions]
+                writer.add_image("mnist_images", img_grid)
+                writer.add_histogram("fc1", model.fc1.weight)
+                writer.add_scalar("Training loss", loss, global_step=step)
+                writer.add_scalar(
+                    "Training Accuracy", running_train_acc, global_step=step
+                )
+
+                if batch_idx == 230:
+                    writer.add_embedding(
+                        features,
+                        metadata=class_labels,
+                        label_img=data,
+                        global_step=batch_idx,
+                    )
+                step += 1
+
+            writer.add_hparams(
+                {"lr": learning_rate, "bsize": batch_size},
+                {
+                    "accuracy": sum(accuracies) / len(accuracies),
+                    "loss": sum(losses) / len(losses),
+                },
+            )

ML/Pytorch/Basics/pytorch_transforms.py

@@ -0,0 +1,155 @@
+"""
+Shows a small example of how to use transformations (perhaps unecessarily many)
+on CIFAR10 dataset and training on a small CNN toy network.
+
+Video explanation: https://youtu.be/Zvd276j9sZ8
+Got any questions leave a comment I'm pretty good at responding on youtube
+
+Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
+*    2020-04-09 Initial coding
+"""
+
+# Imports
+import torch
+import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
+import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
+import torch.nn.functional as F  # All functions that don't have any parameters
+from torch.utils.data import (
+    DataLoader,
+)  # Gives easier dataset managment and creates mini batches
+import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
+import torchvision.transforms as transforms  # Transformations we can perform on our dataset
+
+# Simple CNN
+class CNN(nn.Module):
+    def __init__(self, in_channels, num_classes):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(
+            in_channels=in_channels,
+            out_channels=8,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.pool = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))
+        self.conv2 = nn.Conv2d(
+            in_channels=8,
+            out_channels=16,
+            kernel_size=(3, 3),
+            stride=(1, 1),
+            padding=(1, 1),
+        )
+        self.fc1 = nn.Linear(16 * 8 * 8, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.pool(x)
+        x = F.relu(self.conv2(x))
+        x = self.pool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+
+        return x
+
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Hyperparameters
+learning_rate = 1e-4
+batch_size = 64
+num_epochs = 5
+
+
+# Load pretrain model & modify it
+model = CNN(in_channels=3, num_classes=10)
+model.classifier = nn.Sequential(nn.Linear(512, 100), nn.ReLU(), nn.Linear(100, 10))
+model.to(device)
+
+# Load Data
+my_transforms = transforms.Compose(
+    [  # Compose makes it possible to have many transforms
+        transforms.Resize((36, 36)),  # Resizes (32,32) to (36,36)
+        transforms.RandomCrop((32, 32)),  # Takes a random (32,32) crop
+        transforms.ColorJitter(brightness=0.5),  # Change brightness of image
+        transforms.RandomRotation(
+            degrees=45
+        ),  # Perhaps a random rotation from -45 to 45 degrees
+        transforms.RandomHorizontalFlip(
+            p=0.5
+        ),  # Flips the image horizontally with probability 0.5
+        transforms.RandomVerticalFlip(
+            p=0.05
+        ),  # Flips image vertically with probability 0.05
+        transforms.RandomGrayscale(p=0.2),  # Converts to grayscale with probability 0.2
+        transforms.ToTensor(),  # Finally converts PIL image to tensor so we can train w. pytorch
+        transforms.Normalize(
+            mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]
+        ),  # Note: these values aren't optimal
+    ]
+)
+
+
+train_dataset = datasets.CIFAR10(
+    root="dataset/", train=True, transform=my_transforms, download=True
+)
+train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+
+# Train Network
+for epoch in range(num_epochs):
+    losses = []
+
+    for batch_idx, (data, targets) in enumerate(train_loader):
+        # Get data to cuda if possible
+        data = data.to(device=device)
+        targets = targets.to(device=device)
+
+        # forward
+        scores = model(data)
+        loss = criterion(scores, targets)
+
+        losses.append(loss.item())
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+
+        # gradient descent or adam step
+        optimizer.step()
+
+    print(f"Cost at epoch {epoch} is {sum(losses)/len(losses):.5f}")
+
+# Check accuracy on training & test to see how good our model
+
+
+def check_accuracy(loader, model):
+    if loader.dataset.train:
+        print("Checking accuracy on training data")
+    else:
+        print("Checking accuracy on test data")
+
+    num_correct = 0
+    num_samples = 0
+    model.eval()
+
+    with torch.no_grad():
+        for x, y in loader:
+            x = x.to(device=device)
+            y = y.to(device=device)
+
+            scores = model(x)
+            _, predictions = scores.max(1)
+            num_correct += (predictions == y).sum()
+            num_samples += predictions.size(0)
+
+        print(
+            f"Got {num_correct} / {num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}"
+        )
+
+    model.train()
+
+
+check_accuracy(train_loader, model)

ML/Pytorch/Basics/set_deterministic_behavior/pytorch_set_seeds.py

@@ -0,0 +1,15 @@
+import random, torch, os, numpy as np
+
+def seed_everything(seed=42):
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+seed_everything()
+
+# Do training etc after running seed_everything