The CIFAR-10 dataset (Canadian Institute For Advanced Research) is a collection of images that are commonly used to train machine learning and computer vision algorithms. It is one of the most widely used datasets for machine learning research. The CIFAR-10 dataset contains 60,000 32x32 color images in 10 different classes. The 10 different classes represent airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks. There are 6,000 images of each class.
import numpy as np
import pandas as pd
from packaging import version
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error as MSE
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPool2D, BatchNormalization, Dropout, Flatten, Dense
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.preprocessing import image
from tensorflow.keras.utils import to_categorical
2022-11-20 10:06:55.528177: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
%matplotlib inline
np.set_printoptions(precision=3, suppress=True)
print("This notebook requires TensorFlow 2.0 or above")
print("TensorFlow version: ", tf.__version__)
assert version.parse(tf.__version__).release[0] >=2
This notebook requires TensorFlow 2.0 or above TensorFlow version: 2.10.0
print("Keras version: ", keras.__version__)
Keras version: 2.10.0
# from google.colab import drive
# drive.mount('/content/gdrive')
def get_three_classes(x, y):
def indices_of(class_id):
indices, _ = np.where(y == float(class_id))
return indices
indices = np.concatenate([indices_of(0), indices_of(1), indices_of(2)], axis=0)
x = x[indices]
y = y[indices]
count = x.shape[0]
indices = np.random.choice(range(count), count, replace=False)
x = x[indices]
y = y[indices]
y = tf.keras.utils.to_categorical(y)
return x, y
def show_random_examples(x, y, p):
indices = np.random.choice(range(x.shape[0]), 10, replace=False)
x = x[indices]
y = y[indices]
p = p[indices]
plt.figure(figsize=(10, 5))
for i in range(10):
plt.subplot(2, 5, i + 1)
plt.imshow(x[i])
plt.xticks([])
plt.yticks([])
col = 'green' if np.argmax(y[i]) == np.argmax(p[i]) else 'red'
plt.xlabel(class_names_preview[np.argmax(p[i])], color=col)
plt.show()
def plot_history(history):
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(16, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
def print_validation_report(y_test, predictions):
print("Classification Report")
print(classification_report(y_test, predictions))
print('Accuracy Score: {}'.format(accuracy_score(y_test, predictions)))
print('Root Mean Square Error: {}'.format(np.sqrt(MSE(y_test, predictions))))
def plot_confusion_matrix(y_true, y_pred):
mtx = confusion_matrix(y_true, y_pred)
fig, ax = plt.subplots(figsize=(8,8))
sns.heatmap(mtx, annot=True, fmt='d', linewidths=.75, cbar=False, ax=ax,cmap='Blues',linecolor='white')
# square=True,
plt.ylabel('true label')
plt.xlabel('predicted label')
The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images.
The dataset is divided into five training batches and one test batch, each with 10000 images. The test batch contains exactly 1000 randomly-selected images from each class. The training batches contain the remaining images in random order, but some training batches may contain more images from one class than another. Between them, the training batches contain exactly 5000 images from each class.
(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()
print('train_images:\t{}'.format(x_train.shape))
print('train_labels:\t{}'.format(y_train.shape))
print('test_images:\t\t{}'.format(x_test.shape))
print('test_labels:\t\t{}'.format(y_test.shape))
train_images: (50000, 32, 32, 3) train_labels: (50000, 1) test_images: (10000, 32, 32, 3) test_labels: (10000, 1)
print("First ten labels training dataset:\n {}\n".format(y_train[0:10]))
print("This output the numeric label, need to convert to item description")
First ten labels training dataset: [[6] [9] [9] [4] [1] [1] [2] [7] [8] [3]] This output the numeric label, need to convert to item description
(train_images, train_labels),(test_images, test_labels)= tf.keras.datasets.cifar10.load_data()
x_preview, y_preview = get_three_classes(train_images, train_labels)
x_preview, y_preview = get_three_classes(test_images, test_labels)
class_names_preview = ['aeroplane', 'car', 'bird']
show_random_examples(x_preview, y_preview, y_preview)
The labels are an array of integers, ranging from 0 to 9. These correspond to the class of clothing the image represents:
| Label | Class_ |
|---|---|
| 0 | airplane |
| 1 | automobile |
| 2 | bird |
| 3 | cat |
| 4 | deer |
| 5 | dog |
| 6 | frog |
| 7 | horse |
| 8 | ship |
| 9 | truck |
class_names = ['airplane'
,'automobile'
,'bird'
,'cat'
,'deer'
,'dog'
,'frog'
,'horse'
,'ship'
,'truck']
x_train_split, x_valid_split, y_train_split, y_valid_split = train_test_split(x_train
,y_train
,test_size=.1
,random_state=42
,shuffle=True)
print(x_train_split.shape, x_valid_split.shape, x_test.shape)
(45000, 32, 32, 3) (5000, 32, 32, 3) (10000, 32, 32, 3)
The images are 28x28 NumPy arrays, with pixel values ranging from 0 to 255
x_train_norm = x_train_split/255
x_valid_norm = x_valid_split/255
x_test_norm = x_test/255
We use a Sequential class defined in Keras to create our model. The first 9 layers Conv2D MaxPooling handle feature learning. The last 3 layers, handle classification
model = Sequential([
Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), activation=tf.nn.relu,input_shape=x_train_norm.shape[1:]),
MaxPool2D((2, 2),strides=2),
Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), activation=tf.nn.relu),
MaxPool2D((2, 2),strides=2),
Flatten(),
Dense(units=10, activation=tf.nn.softmax)
])
2022-11-20 10:07:15.988277: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 30, 30, 128) 3584
max_pooling2d (MaxPooling2D (None, 15, 15, 128) 0
)
conv2d_1 (Conv2D) (None, 13, 13, 256) 295168
max_pooling2d_1 (MaxPooling (None, 6, 6, 256) 0
2D)
flatten (Flatten) (None, 9216) 0
dense (Dense) (None, 10) 92170
=================================================================
Total params: 390,922
Trainable params: 390,922
Non-trainable params: 0
_________________________________________________________________
keras.utils.plot_model(model, "CIFAR10.png", show_shapes=True)
You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.
In addition to setting up our model architecture, we also need to define which algorithm should the model use in order to optimize the weights and biases as per the given data. We will use stochastic gradient descent.
We also need to define a loss function. Think of this function as the difference between the predicted outputs and the actual outputs given in the dataset. This loss needs to be minimised in order to have a higher model accuracy. That's what the optimization algorithm essentially does - it minimises the loss during model training. For our multi-class classification problem, categorical cross entropy is commonly used.
Finally, we will use the accuracy during training as a metric to keep track of as the model trains.
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['accuracy'])
history = model.fit(x_train_norm
,y_train_split
,epochs=200
,batch_size=64
,validation_data=(x_valid_norm, y_valid_split)
,callbacks=[
tf.keras.callbacks.ModelCheckpoint("CNN_model.h5",save_best_only=True,save_weights_only=False)
,tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=3),
]
)
Epoch 1/200 704/704 [==============================] - 72s 101ms/step - loss: 1.4379 - accuracy: 0.4913 - val_loss: 1.1596 - val_accuracy: 0.5950 Epoch 2/200 704/704 [==============================] - 77s 109ms/step - loss: 1.0776 - accuracy: 0.6283 - val_loss: 1.0745 - val_accuracy: 0.6178 Epoch 3/200 704/704 [==============================] - 81s 115ms/step - loss: 0.9401 - accuracy: 0.6767 - val_loss: 0.9777 - val_accuracy: 0.6580 Epoch 4/200 704/704 [==============================] - 88s 125ms/step - loss: 0.8501 - accuracy: 0.7087 - val_loss: 0.9442 - val_accuracy: 0.6678 Epoch 5/200 704/704 [==============================] - 91s 129ms/step - loss: 0.7673 - accuracy: 0.7383 - val_loss: 0.8734 - val_accuracy: 0.6966 Epoch 6/200 704/704 [==============================] - 104s 148ms/step - loss: 0.6924 - accuracy: 0.7634 - val_loss: 0.8654 - val_accuracy: 0.7044 Epoch 7/200 704/704 [==============================] - 93s 133ms/step - loss: 0.6405 - accuracy: 0.7798 - val_loss: 0.8951 - val_accuracy: 0.6938 Epoch 8/200 704/704 [==============================] - 89s 127ms/step - loss: 0.5811 - accuracy: 0.7999 - val_loss: 0.8626 - val_accuracy: 0.7126 Epoch 9/200 704/704 [==============================] - 94s 133ms/step - loss: 0.5319 - accuracy: 0.8181 - val_loss: 0.8755 - val_accuracy: 0.7164 Epoch 10/200 704/704 [==============================] - 90s 128ms/step - loss: 0.4871 - accuracy: 0.8324 - val_loss: 0.9042 - val_accuracy: 0.7128 Epoch 11/200 704/704 [==============================] - 88s 126ms/step - loss: 0.4444 - accuracy: 0.8463 - val_loss: 0.9862 - val_accuracy: 0.6976 Epoch 12/200 704/704 [==============================] - 91s 129ms/step - loss: 0.4041 - accuracy: 0.8627 - val_loss: 0.9286 - val_accuracy: 0.7188 Epoch 13/200 704/704 [==============================] - 93s 132ms/step - loss: 0.3679 - accuracy: 0.8730 - val_loss: 0.9858 - val_accuracy: 0.7148 Epoch 14/200 704/704 [==============================] - 96s 137ms/step - loss: 0.3358 - accuracy: 0.8839 - val_loss: 1.0150 - val_accuracy: 0.7172 Epoch 15/200 704/704 [==============================] - 109s 154ms/step - loss: 0.3092 - accuracy: 0.8914 - val_loss: 1.0759 - val_accuracy: 0.7080
In order to ensure that this is not a simple "memorization" by the machine, we should evaluate the performance on the test set. This is easy to do, we simply use the evaluate method on our model.
model = tf.keras.models.load_model("CNN_model.h5")
print(f"Test acc: {model.evaluate(x_test_norm, y_test)[1]:.3f}")
313/313 [==============================] - 5s 15ms/step - loss: 0.8677 - accuracy: 0.7213 Test acc: 0.721
preds = model.predict(x_test_norm)
print('shape of preds: ', preds.shape)
313/313 [==============================] - 5s 16ms/step shape of preds: (10000, 10)
We use Matplotlib to create 2 plots--displaying the training and validation loss (resp. accuracy) for each (training) epoch side by side.
history_dict = history.history
history_dict.keys()
dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])
history_df=pd.DataFrame(history_dict)
history_df.tail().round(3)
| loss | accuracy | val_loss | val_accuracy | |
|---|---|---|---|---|
| 10 | 0.444 | 0.846 | 0.986 | 0.698 |
| 11 | 0.404 | 0.863 | 0.929 | 0.719 |
| 12 | 0.368 | 0.873 | 0.986 | 0.715 |
| 13 | 0.336 | 0.884 | 1.015 | 0.717 |
| 14 | 0.309 | 0.891 | 1.076 | 0.708 |
plot_history(history)
Using both sklearn.metrics. Then we visualize the confusion matrix and see what that tells us.
pred1= model.predict(x_test_norm)
pred1=np.argmax(pred1, axis=1)
313/313 [==============================] - 5s 17ms/step
print_validation_report(y_test, pred1)
Classification Report
precision recall f1-score support
0 0.76 0.76 0.76 1000
1 0.83 0.83 0.83 1000
2 0.65 0.62 0.63 1000
3 0.49 0.61 0.54 1000
4 0.70 0.65 0.67 1000
5 0.64 0.63 0.63 1000
6 0.77 0.81 0.79 1000
7 0.79 0.75 0.77 1000
8 0.87 0.75 0.81 1000
9 0.78 0.82 0.80 1000
accuracy 0.72 10000
macro avg 0.73 0.72 0.72 10000
weighted avg 0.73 0.72 0.72 10000
Accuracy Score: 0.7213
Root Mean Square Error: 2.2032702966272657
plot_confusion_matrix(y_test,pred1)
model = tf.keras.models.load_model('CNN_model.h5')
preds = model.predict(x_test_norm)
313/313 [==============================] - 5s 15ms/step
preds.shape
(10000, 10)
cm = sns.light_palette((260, 75, 60), input="husl", as_cmap=True)
df = pd.DataFrame(preds[0:20], columns = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'])
df.style.format("{:.2%}").background_gradient(cmap=cm)
| airplane | automobile | bird | cat | deer | dog | frog | horse | ship | truck | |
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0.03% | 0.01% | 0.25% | 86.32% | 0.25% | 6.50% | 5.34% | 0.01% | 1.28% | 0.02% |
| 1 | 2.57% | 44.44% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 52.84% | 0.14% |
| 2 | 8.61% | 39.82% | 0.20% | 3.39% | 0.35% | 0.49% | 0.12% | 0.39% | 40.84% | 5.78% |
| 3 | 96.15% | 0.12% | 0.15% | 0.04% | 0.01% | 0.00% | 0.00% | 0.00% | 3.51% | 0.02% |
| 4 | 0.00% | 0.01% | 1.14% | 1.92% | 39.25% | 0.10% | 57.57% | 0.00% | 0.00% | 0.00% |
| 5 | 0.00% | 0.03% | 0.22% | 9.61% | 2.34% | 3.03% | 83.70% | 1.01% | 0.03% | 0.02% |
| 6 | 0.02% | 97.53% | 0.01% | 0.66% | 0.00% | 0.13% | 0.02% | 0.00% | 0.00% | 1.64% |
| 7 | 0.80% | 0.01% | 9.03% | 1.15% | 3.43% | 0.64% | 83.32% | 1.44% | 0.05% | 0.15% |
| 8 | 0.04% | 0.00% | 1.15% | 90.77% | 1.75% | 4.83% | 0.22% | 1.23% | 0.00% | 0.01% |
| 9 | 0.10% | 77.62% | 0.02% | 0.00% | 0.00% | 0.00% | 0.02% | 0.01% | 0.00% | 22.22% |
| 10 | 50.03% | 0.06% | 7.01% | 9.77% | 13.94% | 12.58% | 0.09% | 0.74% | 5.69% | 0.09% |
| 11 | 0.00% | 0.03% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 99.97% |
| 12 | 0.03% | 0.13% | 9.38% | 37.59% | 0.89% | 46.80% | 3.28% | 1.67% | 0.18% | 0.03% |
| 13 | 0.00% | 0.00% | 0.03% | 0.01% | 0.32% | 0.00% | 0.00% | 99.63% | 0.00% | 0.00% |
| 14 | 0.00% | 0.02% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 99.98% |
| 15 | 0.10% | 0.02% | 0.03% | 0.62% | 0.04% | 0.05% | 70.15% | 0.00% | 29.00% | 0.01% |
| 16 | 0.00% | 0.21% | 0.10% | 5.76% | 0.00% | 93.53% | 0.16% | 0.04% | 0.00% | 0.20% |
| 17 | 0.02% | 0.18% | 0.90% | 24.57% | 2.12% | 6.23% | 2.25% | 63.23% | 0.03% | 0.49% |
| 18 | 0.23% | 1.81% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 0.00% | 95.14% | 2.81% |
| 19 | 0.00% | 0.04% | 0.19% | 0.56% | 0.34% | 0.10% | 98.75% | 0.02% | 0.00% | 0.00% |
(_,_), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()
img = test_images[8010]
img_tensor = image.img_to_array(img)
img_tensor = np.expand_dims(img_tensor, axis=0)
class_names = ['airplane'
,'automobile'
,'bird'
,'cat'
,'deer'
,'dog'
,'frog'
,'horse'
,'ship'
,'truck']
plt.imshow(img, cmap='viridis')
plt.axis('off')
plt.show()
# Extracts the outputs of the top 8 layers:
layer_outputs = [layer.output for layer in model.layers[:8]]
# Creates a model that will return these outputs, given the model input:
activation_model = models.Model(inputs=model.input, outputs=layer_outputs)
activations = activation_model.predict(img_tensor)
len(activations)
1/1 [==============================] - 0s 81ms/step
6
layer_names = []
for layer in model.layers:
layer_names.append(layer.name)
layer_names
['conv2d', 'max_pooling2d', 'conv2d_1', 'max_pooling2d_1', 'flatten', 'dense']
# These are the names of the layers, so can have them as part of our plot
layer_names = ['max_pooling2d', 'max_pooling2d_1']
images_per_row = 16
# Now let's display our feature maps
for layer_name, layer_activation in zip(layer_names, activations):
# This is the number of features in the feature map
n_features = layer_activation.shape[-1]
# The feature map has shape (1, size, size, n_features)
size = layer_activation.shape[1]
# We will tile the activation channels in this matrix
n_cols = n_features // images_per_row
display_grid = np.zeros((size * n_cols, images_per_row * size))
# We'll tile each filter into this big horizontal grid
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,
:, :,
col * images_per_row + row]
# Post-process the feature to make it visually palatable
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[col * size : (col + 1) * size,
row * size : (row + 1) * size] = channel_image
# Display the grid
scale = 1. / size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
plt.grid(False)
plt.imshow(display_grid, aspect='auto', cmap='viridis')
plt.show();
/var/folders/zz/wfk650fx7lx3mmyd6_y3yxgr0000gr/T/ipykernel_31231/1793540733.py:27: RuntimeWarning: invalid value encountered in true_divide channel_image /= channel_image.std()
# Extracts the outputs of all layers:
layer_outputs = [layer.output for layer in model.layers]
# Creates a model that will return these outputs, given the model input:
activation_model = models.Model(inputs=model.input, outputs=layer_outputs)
# Get activation values for the last dense layer
activations = activation_model.predict(x_valid_norm[:3250])
dense_layer_activations = activations[-3]
output_layer_activations = activations[-1]
102/102 [==============================] - 5s 44ms/step
from sklearn.manifold import TSNE
# Reduce the dimension using T-SNE to visualize i n a scatterplot
tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(activations[5])
# Scaling
tsne_results = (tsne_results - tsne_results.min()) / (tsne_results.max() - tsne_results.min())
/Users/apoorvsara/opt/anaconda3/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning: The default initialization in TSNE will change from 'random' to 'pca' in 1.2. warnings.warn( /Users/apoorvsara/opt/anaconda3/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning: The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2. warnings.warn(
[t-SNE] Computing 121 nearest neighbors... [t-SNE] Indexed 3250 samples in 0.011s... [t-SNE] Computed neighbors for 3250 samples in 1.198s... [t-SNE] Computed conditional probabilities for sample 1000 / 3250 [t-SNE] Computed conditional probabilities for sample 2000 / 3250 [t-SNE] Computed conditional probabilities for sample 3000 / 3250 [t-SNE] Computed conditional probabilities for sample 3250 / 3250 [t-SNE] Mean sigma: 0.003501 [t-SNE] KL divergence after 250 iterations with early exaggeration: 60.165848 [t-SNE] KL divergence after 300 iterations: 1.002481
import matplotlib as mpl
cmap = plt.cm.tab10
plt.figure(figsize=(16,10))
scatter = plt.scatter(tsne_results[:,0],tsne_results[:,1], c=y_valid_split[:3250], s=10, cmap=cmap)
plt.legend(handles=scatter.legend_elements()[0], labels=class_names)
plt.axis("off")
plt.show()
