Please cite us if you use the software

Example-1 (Comparison of three different classifiers)

A comparison of a 3 classifiers in scikit-learn on iris dataset. The iris dataset is a classic and very easy multi-class classification dataset.

Environment check

Checking that the notebook is running on Google Colab or not.

import sys
try:
  import google.colab
  !{sys.executable} -m pip -q -q install pycm
except:
  pass

Install scikit-learn

import os
!{sys.executable} -m pip -q -q install scikit-learn
if "Example1_files" not in os.listdir():
    os.mkdir("Example1_files")

Load dataset

from sklearn import datasets
from sklearn.model_selection import train_test_split
from pycm import ConfusionMatrix
iris = datasets.load_iris()
X = iris.data
y = iris.target
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

Classifier 1 (C-Support vector)

from sklearn import svm
classifier_1 = svm.SVC(kernel='linear', C=0.01)

y_pred_1 = classifier_1.fit(X_train, y_train).predict(X_test)

cm1=ConfusionMatrix(y_test, y_pred_1)
cm1.print_matrix()

Predict  0        1        2        
Actual
0        13       0        0        

1        0        10       6        

2        0        0        9        



cm1.print_normalized_matrix()

Predict     0           1           2           
Actual
0           1.0         0.0         0.0         

1           0.0         0.625       0.375       

2           0.0         0.0         1.0         



cm1.Kappa
Out[8]:
0.7673469387755101
cm1.Overall_ACC
Out[9]:
0.8421052631578947
cm1.SOA1  # Landis and Koch benchmark
Out[10]:
'Substantial'
cm1.SOA2  # Fleiss’ benchmark
Out[11]:
'Excellent'
cm1.SOA3  # Altman’s benchmark
Out[12]:
'Good'
cm1.SOA4  # Cicchetti’s benchmark
Out[13]:
'Excellent'
cm1.save_html(os.path.join("Example1_files", "cm1"))
Out[14]:
{'Status': True,
 'Message': '/home/sepandhaghighi/pycm/doc/Example1_files/cm1.html'}

Open File

Classifier 2 (Decision tree)

from sklearn.tree import DecisionTreeClassifier
classifier_2 = DecisionTreeClassifier(max_depth=5)

y_pred_2 = classifier_2.fit(X_train, y_train).predict(X_test)

cm2=ConfusionMatrix(y_test, y_pred_2)
cm2.print_matrix()

Predict  0        1        2        
Actual
0        13       0        0        

1        0        15       1        

2        0        0        9        



cm2.print_normalized_matrix()

Predict      0            1            2            
Actual
0            1.0          0.0          0.0          

1            0.0          0.9375       0.0625       

2            0.0          0.0          1.0          



cm2.Kappa
Out[19]:
0.95978835978836
cm2.Overall_ACC
Out[20]:
0.9736842105263158
cm2.SOA1  # Landis and Koch benchmark
Out[21]:
'Almost Perfect'
cm2.SOA2  # Fleiss’ benchmark
Out[22]:
'Excellent'
cm2.SOA3  # Altman’s benchmark
Out[23]:
'Very Good'
cm2.SOA4  # Cicchetti’s benchmark
Out[24]:
'Excellent'
cm2.save_html(os.path.join("Example1_files","cm2"))
Out[25]:
{'Status': True,
 'Message': '/home/sepandhaghighi/pycm/doc/Example1_files/cm2.html'}

Open File

Classifier 3 (AdaBoost)

from sklearn.ensemble import AdaBoostClassifier
classifier_3 = AdaBoostClassifier()

y_pred_3 = classifier_3.fit(X_train, y_train).predict(X_test)

cm3=ConfusionMatrix(y_test, y_pred_3)
cm3.print_matrix()

Predict  0        1        2        
Actual
0        13       0        0        

1        0        15       1        

2        0        1        8        



cm3.print_normalized_matrix()

Predict       0             1             2             
Actual
0             1.0           0.0           0.0           

1             0.0           0.9375        0.0625        

2             0.0           0.11111       0.88889       



cm3.Kappa
Out[30]:
0.9189765458422176
cm3.Overall_ACC
Out[31]:
0.9473684210526315
cm3.SOA1  # Landis and Koch benchmark
Out[32]:
'Almost Perfect'
cm3.SOA2  # Fleiss’ benchmark
Out[33]:
'Excellent'
cm3.SOA3  # Altman’s benchmark
Out[34]:
'Very Good'
cm3.SOA4  # Cicchetti’s benchmark
Out[35]:
'Excellent'
cm3.save_html(os.path.join("Example1_files", "cm3"))
Out[36]:
{'Status': True,
 'Message': '/home/sepandhaghighi/pycm/doc/Example1_files/cm3.html'}

Open File

How to compare classifiers?

from pycm import Compare

cp = Compare({"C-Support vector": cm1, "Decision tree": cm2, "AdaBoost": cm3})
print(cp)

Best : Decision tree

Rank   Name                Class-Score       Overall-Score
1      Decision tree       0.55556           0.95238
2      AdaBoost            0.51944           0.95238
3      C-Support vector    0.44444           0.80476


cp.save_report(os.path.join("Example1_files", "cp"))
Out[38]:
{'Status': True,
 'Message': '/home/sepandhaghighi/pycm/doc/Example1_files/cp.comp'}

Open File

Web Analytics