Initial commit

ML/Projects/Exploring_MNIST/networks/googLeNet.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class Inception(nn.Module):
+    def __init__(
+        self, in_channels, out1x1, out3x3reduced, out3x3, out5x5reduced, out5x5, outpool
+    ):
+        super().__init__()
+
+        self.branch_1 = BasicConv2d(in_channels, out1x1, kernel_size=1, stride=1)
+
+        self.branch_2 = nn.Sequential(
+            BasicConv2d(in_channels, out3x3reduced, kernel_size=1),
+            BasicConv2d(out3x3reduced, out3x3, kernel_size=3, padding=1),
+        )
+
+        # Is in the original googLeNet paper 5x5 conv but in Inception_v2 it has shown to be
+        # more efficient if you instead do two 3x3 convs which is what I am doing here!
+        self.branch_3 = nn.Sequential(
+            BasicConv2d(in_channels, out5x5reduced, kernel_size=1),
+            BasicConv2d(out5x5reduced, out5x5, kernel_size=3, padding=1),
+            BasicConv2d(out5x5, out5x5, kernel_size=3, padding=1),
+        )
+
+        self.branch_4 = nn.Sequential(
+            nn.MaxPool2d(kernel_size=3, stride=1, padding=1),
+            BasicConv2d(in_channels, outpool, kernel_size=1),
+        )
+
+    def forward(self, x):
+        y1 = self.branch_1(x)
+        y2 = self.branch_2(x)
+        y3 = self.branch_3(x)
+        y4 = self.branch_4(x)
+
+        return torch.cat([y1, y2, y3, y4], 1)
+
+
+class GoogLeNet(nn.Module):
+    def __init__(self, img_channel):
+        super().__init__()
+
+        self.first_layers = nn.Sequential(
+            BasicConv2d(img_channel, 192, kernel_size=3, padding=1)
+        )
+
+        self._3a = Inception(192, 64, 96, 128, 16, 32, 32)
+        self._3b = Inception(256, 128, 128, 192, 32, 96, 64)
+
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+
+        self._4a = Inception(480, 192, 96, 208, 16, 48, 64)
+        self._4b = Inception(512, 160, 112, 224, 24, 64, 64)
+        self._4c = Inception(512, 128, 128, 256, 24, 64, 64)
+        self._4d = Inception(512, 112, 144, 288, 32, 64, 64)
+        self._4e = Inception(528, 256, 160, 320, 32, 128, 128)
+
+        self._5a = Inception(832, 256, 160, 320, 32, 128, 128)
+        self._5b = Inception(832, 384, 192, 384, 48, 128, 128)
+
+        self.avgpool = nn.AvgPool2d(kernel_size=8, stride=1)
+        self.linear = nn.Linear(1024, 10)
+
+    def forward(self, x):
+        out = self.first_layers(x)
+
+        out = self._3a(out)
+        out = self._3b(out)
+        out = self.maxpool(out)
+
+        out = self._4a(out)
+        out = self._4b(out)
+        out = self._4c(out)
+        out = self._4d(out)
+        out = self._4e(out)
+        out = self.maxpool(out)
+
+        out = self._5a(out)
+        out = self._5b(out)
+
+        out = self.avgpool(out)
+        out = out.view(out.size(0), -1)
+        out = self.linear(out)
+
+        return out
+
+
+class BasicConv2d(nn.Module):
+    def __init__(self, in_channels, out_channels, **kwargs):
+        super().__init__()
+        self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)
+        self.bn = nn.BatchNorm2d(out_channels, eps=0.001)
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        return F.relu(x, inplace=True)
+
+
+def test():
+    net = GoogLeNet(1)
+    x = torch.randn(3, 1, 32, 32)
+    y = net(x)
+    print(y.size())
+
+
+# test()

ML/Projects/Exploring_MNIST/networks/import_all_networks.py

@@ -0,0 +1,4 @@
+from networks.vgg import VGG
+from networks.lenet import LeNet
+from networks.resnet import ResNet, residual_template, ResNet50, ResNet101, ResNet152
+from networks.googLeNet import BasicConv2d, Inception, GoogLeNet

ML/Projects/Exploring_MNIST/networks/lenet.py

@@ -0,0 +1,60 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class LeNet(nn.Module):
+    def __init__(self, in_channels, init_weights=True, num_classes=10):
+        super(LeNet, self).__init__()
+
+        self.num_classes = num_classes
+
+        if init_weights:
+            self._initialize_weights()
+
+        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=6, kernel_size=5)
+        self.conv2 = nn.Conv2d(in_channels=6, out_channels=16, kernel_size=5)
+        self.fc1 = nn.Linear(16 * 5 * 5, 120)
+        self.fc2 = nn.Linear(120, 84)
+        self.fc3 = nn.Linear(84, 10)
+
+    def forward(self, x):
+        z1 = self.conv1(x)  # 6 x 28 x 28
+        a1 = F.relu(z1)  # 6 x 28 x 28
+        a1 = F.max_pool2d(a1, kernel_size=2, stride=2)  # 6 x 14 x 14
+        z2 = self.conv2(a1)  # 16 x 10 x 10
+        a2 = F.relu(z2)  # 16 x 10 x 10
+        a2 = F.max_pool2d(a2, kernel_size=2, stride=2)  # 16 x 5 x 5
+        flatten_a2 = a2.view(a2.size(0), -1)
+        z3 = self.fc1(flatten_a2)
+        a3 = F.relu(z3)
+        z4 = self.fc2(a3)
+        a4 = F.relu(z4)
+        z5 = self.fc3(a4)
+        return z5
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.constant_(m.bias, 0)
+
+
+def test_lenet():
+    net = LeNet(1)
+    x = torch.randn(64, 1, 32, 32)
+    y = net(x)
+    print(y.size())
+
+
+test_lenet()

ML/Projects/Exploring_MNIST/networks/resnet.py

@@ -0,0 +1,151 @@
+import torch
+import torch.nn as nn
+
+
+class residual_template(nn.Module):
+    expansion = 4
+
+    def __init__(self, in_channels, out_channels, stride=1, identity_downsample=None):
+        super().__init__()
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(out_channels)
+        self.conv2 = nn.Conv2d(
+            out_channels,
+            out_channels,
+            kernel_size=3,
+            stride=stride,
+            padding=1,
+            bias=False,
+        )
+        self.bn2 = nn.BatchNorm2d(out_channels)
+        self.conv3 = nn.Conv2d(
+            out_channels, out_channels * self.expansion, kernel_size=1, bias=False
+        )
+        self.bn3 = nn.BatchNorm2d(out_channels * self.expansion)
+        self.relu = nn.ReLU(inplace=True)
+        self.identity_downsample = identity_downsample
+        self.stride = stride
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.identity_downsample is not None:
+            residual = self.identity_downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+
+class ResNet(nn.Module):
+    def __init__(self, residual_template, layers, image_channel, num_classes=10):
+        self.in_channels = 64
+        super().__init__()
+
+        self.conv1 = nn.Conv2d(
+            in_channels=image_channel,
+            out_channels=64,
+            kernel_size=3,
+            stride=1,
+            padding=1,
+            bias=False,
+        )
+        self.bn1 = nn.BatchNorm2d(64)
+        self.relu = nn.ReLU(inplace=True)
+        self.layer1 = self._make_layer(
+            residual_template, layers[0], channels=64, stride=1
+        )
+        self.layer2 = self._make_layer(
+            residual_template, layers[1], channels=128, stride=2
+        )
+        self.layer3 = self._make_layer(
+            residual_template, layers[2], channels=256, stride=2
+        )
+        self.layer4 = self._make_layer(
+            residual_template, layers[3], channels=512, stride=2
+        )
+        self.avgpool = nn.AvgPool2d(kernel_size=4, stride=1)
+        self.fc = nn.Linear(512 * residual_template.expansion, num_classes)
+
+        # initialize weights for conv layers, batch layers
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+    def _make_layer(self, residual_template, num_residuals_blocks, channels, stride):
+        identity_downsample = None
+
+        if stride != 1 or self.in_channels != channels * residual_template.expansion:
+            identity_downsample = nn.Sequential(
+                nn.Conv2d(
+                    self.in_channels,
+                    channels * residual_template.expansion,
+                    kernel_size=1,
+                    stride=stride,
+                    bias=False,
+                ),
+                nn.BatchNorm2d(channels * residual_template.expansion),
+            )
+
+        layers = []
+        layers.append(
+            residual_template(self.in_channels, channels, stride, identity_downsample)
+        )
+        self.in_channels = channels * residual_template.expansion
+
+        for i in range(1, num_residuals_blocks):
+            layers.append(residual_template(self.in_channels, channels))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        x = self.avgpool(x)
+        x = x.view(x.size(0), -1)
+        x = self.fc(x)
+
+        return x
+
+
+def ResNet50(img_channel):
+    return ResNet(residual_template, [3, 4, 6, 3], img_channel)
+
+
+def ResNet101(img_channel):
+    return ResNet(residual_template, [3, 4, 23, 3], img_channel)
+
+
+def ResNet152(img_channel):
+    return ResNet(residual_template, [3, 8, 36, 3], img_channel)
+
+
+def test():
+    net = ResNet152(img_channel=1)
+    y = net(torch.randn(64, 1, 32, 32))
+    print(y.size())
+
+
+# test()

ML/Projects/Exploring_MNIST/networks/vgg.py

@@ -0,0 +1,139 @@
+import torch
+import torch.nn as nn
+
+
+VGG_types = {
+    "VGG11": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
+    "VGG13": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
+    "VGG16": [
+        64,
+        64,
+        "M",
+        128,
+        128,
+        "M",
+        256,
+        256,
+        256,
+        "M",
+        512,
+        512,
+        512,
+        "M",
+        512,
+        512,
+        512,
+        "M",
+    ],
+    "VGG19": [
+        64,
+        64,
+        "M",
+        128,
+        128,
+        "M",
+        256,
+        256,
+        256,
+        256,
+        "M",
+        512,
+        512,
+        512,
+        512,
+        "M",
+        512,
+        512,
+        512,
+        512,
+        "M",
+    ],
+}
+
+
+class VGG(nn.Module):
+    def __init__(
+        self, vgg_type, in_channels, init_weights=True, batch_norm=True, num_classes=10
+    ):
+        super().__init__()
+
+        self.batch_norm = batch_norm
+        self.in_channels = in_channels
+
+        self.layout = self.create_architecture(VGG_types[vgg_type])
+        self.fc = nn.Linear(512, num_classes)
+
+        # self.fcs = nn.Sequential(
+        #     nn.Linear(512* 1 * 1, 4096),
+        #     nn.ReLU(inplace = False),
+        #     nn.Dropout(),
+        #     nn.Linear(4096, 4096),
+        #     nn.ReLU(inplace = False),
+        #     nn.Dropout(),
+        #     nn.Linear(4096, num_classes),
+        # )
+
+        if init_weights:
+            self._initialize_weights()
+
+    def forward(self, x):
+        out = self.layout(x)
+        out = out.view(out.size(0), -1)
+        out = self.fc(out)
+
+        return out
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.constant_(m.bias, 0)
+
+    def create_architecture(self, architecture):
+        layers = []
+
+        for x in architecture:
+            if type(x) == int:
+                out_channels = x
+
+                conv2d = nn.Conv2d(
+                    self.in_channels, out_channels, kernel_size=3, padding=1
+                )
+
+                if self.batch_norm:
+                    layers += [
+                        conv2d,
+                        nn.BatchNorm2d(out_channels),
+                        nn.ReLU(inplace=False),
+                    ]
+                else:
+                    layers += [conv2d, nn.ReLU(inplace=False)]
+
+                self.in_channels = out_channels
+
+            elif x == "M":
+                layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
+
+        layers += [nn.AvgPool2d(kernel_size=1, stride=1)]
+
+        return nn.Sequential(*layers)
+
+
+def test():
+    net = VGG("VGG16", 1)
+    x = torch.randn(64, 1, 32, 32)
+    y = net(x)
+    print(y.size())
+
+
+# test()