Initial commit

ML/TensorFlow/CNN_architectures/AlexNet/README.md

@@ -0,0 +1,9 @@
+[Original Paper - ImageNet Classification with Deep Convolutional Neural Networks (2012)](https://www.cs.toronto.edu/~hinton/absps/imagenet.pdf)  
+
+Some questions I had when I was reading the paper
+- [What does the term saturating nonlinearities mean?](https://stats.stackexchange.com/questions/174295/what-does-the-term-saturating-nonlinearities-mean)
+- [What Is Saturating Gradient Problem](https://datascience.stackexchange.com/questions/27665/what-is-saturating-gradient-problem)
+- [Why ReLU is better than the other activation functions](https://datascience.stackexchange.com/questions/23493/why-relu-is-better-than-the-other-activation-functions)
+- [Why does overlapped pooling help reduce overfitting in conv nets?](https://stats.stackexchange.com/questions/283261/why-does-overlapped-pooling-help-reduce-overfitting-in-conv-nets)
+- [Importance of local response normalization in CNN](https://stats.stackexchange.com/questions/145768/importance-of-local-response-normalization-in-cnn)
+- [What Is Local Response Normalization In Convolutional Neural Networks](https://prateekvjoshi.com/2016/04/05/what-is-local-response-normalization-in-convolutional-neural-networks/)

ML/TensorFlow/CNN_architectures/AlexNet/alexnet.py

@@ -0,0 +1,113 @@
+# Tensorflow v2.3.1
+
+"""
+Programmed by the-robot <https://github.com/the-robot>
+"""
+
+from tensorflow.keras.layers import (
+    Conv2D,
+    Dense,
+    Dropout,
+    Flatten,
+    Input,
+    Lambda,
+    MaxPooling2D,
+)
+from tensorflow.keras import Model
+import tensorflow as tf
+import typing
+
+tf.config.run_functions_eagerly(True)
+
+@tf.function
+def AlexNet(input_shape: typing.Tuple[int], classes: int = 1000) -> Model:
+    """
+    Implementation of the AlexNet architecture.
+
+    Arguments:
+    input_shape -- shape of the images of the dataset
+    classes     -- integer, number of classes
+
+    Returns:
+    model       -- a Model() instance in Keras
+
+    Note:
+    when you read the paper, you will notice that the channels (filters) in the diagram is only
+    half of what I have written below. That is because in the diagram, they only showed model for
+    one GPU (I guess for simplicity). However, during the ILSVRC, they run the network across 2 NVIDIA GTA 580 3GB GPUs.
+
+    Also, in paper, they used Local Response Normalization. This can also be done in Keras with Lambda layer.
+    You can also use BatchNormalization layer instead.
+    """
+
+    # convert input shape into tensor
+    X_input = Input(input_shape)
+
+    # NOTE: layer 1-5 is conv-layers
+    # layer 1
+    X = Conv2D(
+        filters = 96,
+        kernel_size = (11, 11),
+        strides = (4, 4),
+        activation = "relu",
+        padding = "same",
+    )(X_input)
+    X = MaxPooling2D(pool_size = (3, 3), strides = (2, 2))(X)
+    X = Lambda(tf.nn.local_response_normalization)(X)
+
+    # layer 2
+    X = Conv2D(
+        filters = 256,
+        kernel_size = (5, 5),
+        strides = (1, 1),
+        activation = "relu",
+        padding = "same",
+    )(X)
+    X = MaxPooling2D(pool_size = (3, 3), strides = (2, 2))(X)
+    X = Lambda(tf.nn.local_response_normalization)(X)
+
+    # layer 3
+    X = Conv2D(
+        filters = 384,
+        kernel_size = (3, 3),
+        strides = (1, 1),
+        activation = "relu",
+        padding = "same",
+    )(X)
+
+    # layer 4
+    X = Conv2D(
+        filters = 384,
+        kernel_size = (3, 3),
+        strides = (1, 1),
+        activation = "relu",
+        padding = "same",
+    )(X)
+
+    # layer 5
+    X = Conv2D(
+        filters = 256,
+        kernel_size = (3, 3),
+        strides = (1, 1),
+        activation = "relu",
+        padding = "same",
+    )(X)
+    X = MaxPooling2D(pool_size = (3, 3), strides = (2, 2))(X)
+    X = Lambda(tf.nn.local_response_normalization)(X)
+
+    # NOTE: layer 6-7 is fully-connected layers
+    # layer 6
+    X = Flatten()(X)
+    X = Dense(units = 4096, activation = 'relu')(X)
+    X = Dropout(0.5)(X)
+
+    # layer 7
+    X = Dense(units = 4096, activation = 'relu')(X)
+    X = Dropout(0.5)(X)
+
+    # layer 8 (classification layer)
+    # use sigmoid if binary classificaton and softmax if multiclass classification
+    X = Dense(units = classes, activation = "softmax")(X)
+
+    model = Model(inputs = X_input, outputs = X, name = "AlexNet")
+    return model

ML/TensorFlow/CNN_architectures/AlexNet/test.py

@@ -0,0 +1,9 @@
+# disable tensorflow debugging messages
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+from alexnet import AlexNet
+
+if __name__ == "__main__":
+    model = AlexNet(input_shape = (224, 224, 3), classes = 1000)
+    model.summary()