Multiclass classification using Perceptron from scratch

In binary perceptron, where $\mathbf{y} \in {-1, +1}$, we used to update our weights only for wrongly classified examples.

Here we have used multiple one vs rest binary classifiers using Perceptron to create a multiclass classification of MNIST data.

Importing libraries

from sklearn import datasets
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt
import random
import numpy as np
import seaborn as sns; sns.set();
import pandas as pd
import math
import gif
import warnings
warnings.filterwarnings('ignore')

Setting up the data

# Setting the seed to ensure reproducibility of experiments
np.random.seed(11)

# One-hot encoding of target label, Y
def one_hot(a):
  b = -1 * np.ones((a.size, a.max()+1))
  b[np.arange(a.size), a] = 1
  return b

# Loading digits datasets
digits = datasets.load_digits()

# One-hot encoding of target label, Y
Y = digits.target
Y = one_hot(Y)

# Adding column of ones to absorb bias b of the hyperplane into X
X = digits.data
bias_ones = np.ones((len(X), 1))
X = np.hstack((X, bias_ones))

Splitting the dataset into train , test and validation

# Train-val-test data
X_train_val, X_test, Y_train_val, Y_test = train_test_split(X, Y, shuffle=True, test_size = 0.2)
X_train, X_val, Y_train, Y_val = train_test_split(X_train_val, Y_train_val, test_size = 0.12517)

print("Training dataset: ", X_train.shape)
print("Validation dataset: ", X_val.shape)
print("Test dataset: ", X_test.shape)
Training dataset:  (1257, 65)
Validation dataset:  (180, 65)
Test dataset:  (360, 65)
sns.reset_orig();

plt.gray()
plt.matshow(digits.images[10])
plt.show();
<Figure size 432x288 with 0 Axes>

Implementing perceptron algorithm for a binary one vs all classifier

The pseudo-code for our perceptron algorithm is as follows:

#Here we define a signum function to use later
def signum(x):
    if x >=0: return 1
    else : return -1

#General function for perceptron 
def Perceptron(X_train,Y_train,epochs,lr):
    epoch = 1
    m = 1
    # One vs Rest is a two class classification problem
    w = np.zeros((X_train.shape[1],1))     #Initialising the weights as 0
    while(m!=0 and epoch <= epochs):       # Iterating till epochs are reached or when error becomes 0
        m = 0 
        for xi,yi in zip(X_train,Y_train):  # Iterating over each sample and the corresponding class label
            y_hat = signum(np.dot(w.T,xi)[0]) 
                                            
            if yi*y_hat <0:                 #Condition for misclassified samples
                w = (w.T + yi*xi).T         ##Updating weights
                m = m + 1                   #Updating error count
        epoch = epoch + 1                   #Increasing epochs for looping
    return w,m

Training the model

weights = np.zeros((X_train.shape[1],Y_train.shape[1])) # Defining a weight matrix (Num_of_features * Num of classes) to store each weight vectors
for i in range(Y_train.shape[1]):
    w,err = Perceptron(X_train,Y_train[:,i],100,1) #Getting the weight vector
    weights[:,i] = w[:,0]                          #Storing into the weight matrix

Fitting the model into validation and then test datasets

#Predicting the classes based on the weight vectors
def predictclass(X_fit,Y_fit,weights):
    predictedclass = np.zeros(X_fit.shape[0])
    for i in range(X_fit.shape[0]):
        for j in range(Y_fit.shape[1]):
            predict = np.dot(weights[:,j],X_fit[i,:]) #Samplewise prediction
            if predict >0:
                predictedclass[i] = j
                break
    return predictedclass
# Function to find the accuracy
def accuracy(Y_val,predictedclass):
    error = 0
    numsamples = Y_val.shape[0]
    for i in range(numsamples):
        Actualclass = Y_val[i,:]
        if Actualclass[int(predictedclass[i])]!=1.0:
             error+=1
    return (1-error/numsamples)
#Array of predicted class for validation set
predictedclass = predictclass(X_val,Y_val,weights)

Finding Accuracies

#Accuracy for validation set
accuracy(Y_val,predictedclass) * 100
91.11111111111111
#Array of predicted class for test set
predictedclass = predictclass(X_test,Y_test,weights)
#Accuracy for test set
accuracy(Y_test,predictedclass) * 100
89.44444444444444