Initial commit

ML/TensorFlow/CNN_architectures/ResNet/README.md

@@ -0,0 +1,7 @@
+[Original Paper - Deep Residual Learning for Image Recognition (2015)](https://arxiv.org/abs/1512.03385)  
+[Related Video](https://www.youtube.com/watch?v=DkNIBBBvcPs&ab_channel=AladdinPersson)
+
+Some questions that came to my mind when I was reading the paper
+
+- [How do bottleneck architectures work in neural networks?](https://stats.stackexchange.com/questions/205150/how-do-bottleneck-architectures-work-in-neural-networks)
+- [What does dotted line mean in ResNet?](https://stats.stackexchange.com/questions/457787/what-does-dotted-line-mean-in-resnet) `refering to Figure 3, 34-layer residual from paper`

ML/TensorFlow/CNN_architectures/ResNet/block.py

@@ -0,0 +1,105 @@
+# Tensorflow v.2.3.1
+
+"""
+Programmed by the-robot <https://github.com/the-robot>
+"""
+
+from tensorflow.keras.layers import (
+    Activation,
+    Add,
+    BatchNormalization,
+    Conv2D,
+)
+import tensorflow as tf
+import typing
+
+@tf.function
+def block(
+    X: tf.Tensor,
+    kernel_size: int,
+    filters: typing.List[int],
+    stage_no: int,
+    block_name: str,
+    is_conv_layer: bool = False,
+    stride: int = 2
+) -> tf.Tensor:
+    """
+    Block for residual network.
+
+    Arguments:
+    X             -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
+    kernel_size   -- integer, specifying the shape of the middle CONV's window for the main path
+    filters       -- python list of integers, defining the number of filters in the CONV layers of the main path
+    stage_no      -- integer, used to name the layers, depending on their position in the network
+    block_name    -- string/character, used to name the layers, depending on their position in the network
+    is_conv_layer -- to identiy if identity downsample is needed
+    stride        -- integer specifying the stride to be used
+    
+    Returns:
+    X             -- output of the identity block, tensor of shape (n_H, n_W, n_C)
+    """
+
+    # names
+    conv_name_base = "res" + str(stage_no) + block_name + "_branch"
+    bn_name_base = "bn" + str(stage_no) + block_name + "_branch"
+
+    # filters
+    F1, F2, F3 = filters
+
+    # save the input value for shortcut.
+    X_shortcut = X
+
+    #  First component
+    # NOTE: if conv_layer, you need to do downsampling
+    X = Conv2D(
+        filters = F1,
+        kernel_size = (1, 1),
+        strides = (stride, stride) if is_conv_layer else (1, 1),
+        padding = "valid",
+        name = conv_name_base + "2a",
+        kernel_initializer = "glorot_uniform",
+    )(X)
+    X = BatchNormalization(axis = 3, name = bn_name_base + "2a")(X)
+    X = Activation("relu")(X)
+
+    # Second component
+    X = Conv2D(
+        filters = F2,
+        kernel_size = (kernel_size, kernel_size),
+        strides = (1, 1),
+        padding = "same",
+        name = conv_name_base + "2b",
+        kernel_initializer = "glorot_uniform",
+    )(X)
+    X = BatchNormalization(axis = 3, name = bn_name_base + "2b")(X)
+    X = Activation("relu")(X)
+
+    # Third component
+    X = Conv2D(
+        filters = F3,
+        kernel_size = (1, 1),
+        strides = (1, 1),
+        padding = "valid",
+        name = conv_name_base + "2c",
+        kernel_initializer = "glorot_uniform",
+    )(X)
+    X = BatchNormalization(axis = 3, name = bn_name_base + "2c")(X)
+
+    # NOTE: if is_conv_layer, you need to do downsampling the X_shortcut to match the output (X) channel
+    #       so it can be added together
+    if is_conv_layer:
+        X_shortcut = Conv2D(
+            filters = F3,
+            kernel_size = (1, 1),
+            strides = (stride, stride),
+            padding = "valid",
+            name = conv_name_base + "1",
+            kernel_initializer = "glorot_uniform",
+        )(X_shortcut)
+        X_shortcut = BatchNormalization(axis = 3, name = bn_name_base + "1")(X_shortcut)
+
+    # Shortcut value
+    X = Add()([X, X_shortcut])
+    X = Activation("relu")(X)
+
+    return X

ML/TensorFlow/CNN_architectures/ResNet/resnet.py

@@ -0,0 +1,157 @@
+# Tensorflow v.2.3.1
+
+"""
+Programmed by the-robot <https://github.com/the-robot>
+"""
+
+from block import block
+
+from tensorflow.keras.layers import (
+    Activation,
+    AveragePooling2D,
+    BatchNormalization,
+    Conv2D,
+    Dense,
+    Flatten,
+    Input,
+    MaxPooling2D,
+    ZeroPadding2D,
+)
+from tensorflow.keras import Model
+import tensorflow as tf
+import typing
+
+tf.config.run_functions_eagerly(True)
+
+@tf.function
+def ResNet(name: str, layers: typing.List[int], input_shape: typing.Tuple[int] = (64, 64, 3), classes: int = 6) -> Model:
+    """
+    Implementation of the popular ResNet architecture.
+
+    Arguments:
+    name        -- name of the architecture
+    layers      -- number of blocks per layer
+    input_shape -- shape of the images of the dataset
+    classes     -- integer, number of classes
+
+    Returns:
+    model       -- a Model() instance in Keras
+
+
+    Model Architecture:
+    Resnet50:
+        CONV2D -> BATCHNORM -> RELU -> MAXPOOL  // conv1
+            -> CONVBLOCK -> IDBLOCK * 2         // conv2_x
+            -> CONVBLOCK -> IDBLOCK * 3         // conv3_x
+            -> CONVBLOCK -> IDBLOCK * 5         // conv4_x
+            -> CONVBLOCK -> IDBLOCK * 2         // conv5_x
+            -> AVGPOOL
+            -> TOPLAYER
+
+    Resnet101:
+        CONV2D -> BATCHNORM -> RELU -> MAXPOOL  // conv1
+            -> CONVBLOCK -> IDBLOCK * 2         // conv2_x
+            -> CONVBLOCK -> IDBLOCK * 3         // conv3_x
+            -> CONVBLOCK -> IDBLOCK * 22        // conv4_x
+            -> CONVBLOCK -> IDBLOCK * 2         // conv5_x
+            -> AVGPOOL
+            -> TOPLAYER
+
+    Resnet152:
+        CONV2D -> BATCHNORM -> RELU -> MAXPOOL  // conv1
+            -> CONVBLOCK -> IDBLOCK * 2         // conv2_x
+            -> CONVBLOCK -> IDBLOCK * 7         // conv3_x
+            -> CONVBLOCK -> IDBLOCK * 35        // conv4_x
+            -> CONVBLOCK -> IDBLOCK * 2         // conv5_x
+            -> AVGPOOL
+            -> TOPLAYER
+    """
+
+    # get layers (layer1 is always the same so no need to provide)
+    layer2, layer3, layer4, layer5 = layers
+
+    # convert input shape into tensor
+    X_input = Input(input_shape)
+
+    # zero-padding
+    X = ZeroPadding2D((3, 3))(X_input)
+
+    # conv1
+    X = Conv2D(
+        filters = 64,
+        kernel_size = (7, 7),
+        strides = (2, 2),
+        name = "conv1",
+        kernel_initializer = "glorot_uniform",
+    )(X)
+    X = BatchNormalization(axis = 3, name = "bn_conv1")(X)
+    X = Activation("relu")(X)
+    X = MaxPooling2D((3, 3), strides = (2, 2))(X)
+
+    # conv2_x
+    X = make_layer(X, layers = layer2, kernel_size = 3, filters = [64, 64, 256], stride = 1, stage_no = 2)
+
+    # conv3_x
+    X = make_layer(X, layers = layer3, kernel_size = 3, filters = [128, 128, 512], stride = 2, stage_no = 3)
+
+    # conv4_x
+    X = make_layer(X, layers = layer4, kernel_size = 3, filters = [256, 256, 1024], stride = 2, stage_no = 4)
+
+    # conv5_x
+    X = make_layer(X, layers = layer5, kernel_size = 3, filters = [512, 512, 2048], stride = 1, stage_no = 5)
+
+    # average pooling
+    X = AveragePooling2D((2, 2), name = "avg_pool")(X)
+
+    # output layer
+    X = Flatten()(X)
+    X = Dense(
+        classes,
+        activation = "softmax",
+        name="fc" + str(classes),
+        kernel_initializer = "glorot_uniform"
+    )(X)
+
+    model = Model(inputs = X_input, outputs = X, name = name)
+    return model
+
+def make_layer(X: tf.Tensor, layers: int, kernel_size: int, filters: typing.List[int], stride: int, stage_no: int) -> tf.Tensor:
+    """
+    Method to create one conv-identity layer for ResNet.
+
+    Arguments:
+    X           -- input tensor
+    layers      -- number of blocks per layer
+    kernel_size -- size of the kernel for the block
+    filters     -- number of filters/channels
+    stride      -- number of stride for downsampling the input
+    stage_no    -- stage number just to name the layer
+
+    Returns:
+    X           -- output tensor
+    """
+
+    # create convolution block
+    X = block(
+        X,
+        kernel_size = kernel_size,
+        filters = filters,
+        stage_no = stage_no,
+        block_name = "a",
+        is_conv_layer = True,
+        stride = stride
+    )
+
+    # create identity block
+    block_name_ordinal = ord("b")
+    for _ in range(layers - 1):
+        X = block(
+            X,
+            kernel_size = kernel_size,
+            filters =  filters,
+            stage_no = stage_no,
+            block_name = chr(block_name_ordinal)
+        )
+        block_name_ordinal += 1
+
+    return X

ML/TensorFlow/CNN_architectures/ResNet/test.py

@@ -0,0 +1,10 @@
+# disable tensorflow debugging messages
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+from resnet import ResNet
+
+if __name__ == "__main__":
+    # test ResNet50
+    model = ResNet(name = "Resnet50", layers = [3, 4, 6, 3], input_shape = (64, 64, 3), classes = 6)
+    model.summary()