{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "303f8e1e-7792-4bb3-9ee5-6de3dd9ebae7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c44f5ca5-ea4b-4cc5-93b6-6f5cdb2d6622",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score,precision_recall_fscore_support\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "import time\n",
    "from river import stream\n",
    "from statistics import mode\n",
    "from statistics import mode\n",
    "from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from statistics import mode\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LassoCV\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, roc_curve, auc, log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import joblib\n",
    "from joblib import load\n",
    "import os\n",
    "from sklearn.preprocessing import PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "285d3134-80b0-4957-9494-28675ea6a254",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/CHDCNT.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9b1c5a78-cb30-434c-b0a6-6d35de5933e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CAN ID</th>\n",
       "      <th>DLC</th>\n",
       "      <th>DataOne</th>\n",
       "      <th>DataTwo</th>\n",
       "      <th>DataThree</th>\n",
       "      <th>DataFour</th>\n",
       "      <th>DataFive</th>\n",
       "      <th>DataSix</th>\n",
       "      <th>DataSeven</th>\n",
       "      <th>DataEight</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1107</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>224</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1157</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>593</td>\n",
       "      <td>8</td>\n",
       "      <td>201</td>\n",
       "      <td>3</td>\n",
       "      <td>130</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>213</td>\n",
       "      <td>167</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1170</td>\n",
       "      <td>8</td>\n",
       "      <td>64</td>\n",
       "      <td>254</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>854</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>128</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>246</td>\n",
       "      <td>1</td>\n",
       "      <td>133</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>232</td>\n",
       "      <td>200</td>\n",
       "      <td>108</td>\n",
       "      <td>23</td>\n",
       "      <td>126</td>\n",
       "      <td>183</td>\n",
       "      <td>165</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>871</td>\n",
       "      <td>8</td>\n",
       "      <td>146</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>215</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>304</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>128</td>\n",
       "      <td>84</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>97</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>1136</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>84</td>\n",
       "      <td>80</td>\n",
       "      <td>84</td>\n",
       "      <td>177</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CAN ID  DLC  DataOne  DataTwo  DataThree  DataFour  DataFive  DataSix  \\\n",
       "0        1107    5        0      136        141         0       224      256   \n",
       "1        1157    4        3        0          0         0       256      256   \n",
       "2         593    8      201        3        130        76         0      213   \n",
       "3        1170    8       64      254         15         0         0        0   \n",
       "4         854    8        0        0          0       128        19        0   \n",
       "...       ...  ...      ...      ...        ...       ...       ...      ...   \n",
       "99995     832    8        0        0          0        36       246        1   \n",
       "99996     832    8       31      232        200       108        23      126   \n",
       "99997     871    8      146      128          0         0         0        0   \n",
       "99998     304    8        4      128         84       128         0        0   \n",
       "99999    1136    8       21       65          1         4        84       80   \n",
       "\n",
       "       DataSeven  DataEight  Label  \n",
       "0            256        256      0  \n",
       "1            256        256      0  \n",
       "2            167        126      0  \n",
       "3              0        136      0  \n",
       "4              0          0      0  \n",
       "...          ...        ...    ...  \n",
       "99995        133         48      0  \n",
       "99996        183        165      2  \n",
       "99997        215         10      0  \n",
       "99998          7         97      0  \n",
       "99999         84        177      0  \n",
       "\n",
       "[100000 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6d328b89-f7c8-4048-84cf-8c7f70456b3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    89407\n",
       "1     5349\n",
       "2     2157\n",
       "3     1919\n",
       "4     1168\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bfb1fa87-d598-45cd-8cc3-698e491690c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(['Label'],axis=1)\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, train_size = 0.8, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fb4bccb-f7d6-4885-b674-8028f6ef44b6",
   "metadata": {},
   "source": [
    "0 Normal \\\n",
    "1 DOS\\\n",
    "2 Fuzzing\\\n",
    "3 Replay\\\n",
    "4 Spoofing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a95f5489-6129-4f55-bf4c-4ec70f96a932",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote=SMOTE(n_jobs=-1,sampling_strategy={2:4000,3:4000,4:4000})\n",
    "#X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ddbbdac6-0089-48eb-8df6-bc74ea0f2995",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "31ef3d74-a076-4be5-ae07-bf22204ba6e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    71467\n",
       "1     4324\n",
       "4     4000\n",
       "2     4000\n",
       "3     4000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48924cd5-dafd-4d75-85a6-b278efe8c2ed",
   "metadata": {},
   "source": [
    "# ModelTest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "788f0818-9efe-4cfe-99cc-b8d19e1d3084",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "82198078-74b1-47c9-841d-7a2432484756",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of RF: 0.9848\n",
      "Precision of RF: 0.9818273094414781\n",
      "Recall of RF: 0.9848\n",
      "F1-score of RF: 0.9821159501025651\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99     17940\n",
      "           1       1.00      1.00      1.00      1025\n",
      "           2       1.00      1.00      1.00       458\n",
      "           3       0.68      0.32      0.43       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.98     20000\n",
      "   macro avg       0.93      0.86      0.88     20000\n",
      "weighted avg       0.98      0.98      0.98     20000\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29204 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Random Forest training and prediction\n",
    "rf = RandomForestClassifier(random_state = 0)\n",
    "rf.fit(X_train,y_train.astype('int')) \n",
    "rf_score=rf.score(X_test,y_test.astype('int'))\n",
    "y_predict=rf.predict(X_test)\n",
    "y_true=y_test.astype('int')\n",
    "print('Accuracy of RF: '+ str(rf_score))\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') \n",
    "print('Precision of RF: '+(str(precision)))\n",
    "print('Recall of RF: '+(str(recall)))\n",
    "print('F1-score of RF: '+(str(fscore)))\n",
    "print(classification_report(y_true,y_predict))\n",
    "cm=confusion_matrix(y_true,y_predict)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1736d191-c938-4483-937d-160f055f6898",
   "metadata": {},
   "source": [
    "## ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b688206c-1bb7-486d-8583-ad58c488a2b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of ET: 0.9849\n",
      "Precision of ET: 0.9819104170630153\n",
      "Recall of ET: 0.9849\n",
      "F1-score of ET: 0.9819707699898821\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99     17940\n",
      "           1       1.00      1.00      1.00      1025\n",
      "           2       1.00      1.00      1.00       458\n",
      "           3       0.69      0.30      0.42       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.98     20000\n",
      "   macro avg       0.94      0.86      0.88     20000\n",
      "weighted avg       0.98      0.98      0.98     20000\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29336 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 4.12 s\n",
      "Wall time: 4.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Extra trees training and prediction\n",
    "et = ExtraTreesClassifier(random_state = 0)\n",
    "et.fit(X_train,y_train.astype('int')) \n",
    "et_score=et.score(X_test,y_test.astype('int'))\n",
    "y_predict=et.predict(X_test)\n",
    "y_true=y_test.astype('int')\n",
    "print('Accuracy of ET: '+ str(et_score))\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') \n",
    "print('Precision of ET: '+(str(precision)))\n",
    "print('Recall of ET: '+(str(recall)))\n",
    "print('F1-score of ET: '+(str(fscore)))\n",
    "print(classification_report(y_true,y_predict))\n",
    "cm=confusion_matrix(y_true,y_predict)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1d28ac3-0df3-450d-abb2-84da6912cad4",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "01c3da5f-6275-4906-91df-b32ec2a2fd35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99     17940\n",
      "           1       0.99      1.00      1.00      1025\n",
      "           2       1.00      0.99      1.00       458\n",
      "           3       0.64      0.28      0.39       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.98     20000\n",
      "   macro avg       0.92      0.85      0.87     20000\n",
      "weighted avg       0.98      0.98      0.98     20000\n",
      "\n",
      "Accuracy of XGBoost: 0.98355\n",
      "Precision of XGBoost: 0.9799960384892294\n",
      "Recall of XGBoost: 0.98355\n",
      "Average F1 of XGBoost: 0.9804884773180947\n",
      "F1 of XGBoost for each type of attack: [0.99088163 0.99708171 0.99561404 0.38771593 1.        ]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29364 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 13.9 s\n",
      "Wall time: 1.05 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the XGBoost algorithm\n",
    "import xgboost as xgb\n",
    "xg = xgb.XGBClassifier()\n",
    "X_train_x = X_train.values\n",
    "X_test_x = X_test.values\n",
    "xg.fit(X_train_x, y_train)\n",
    "y_pred = xg.predict(X_test_x)\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(\"Accuracy of XGBoost: \"+ str(accuracy_score(y_test, y_pred)))\n",
    "print(\"Precision of XGBoost: \"+ str(precision_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Recall of XGBoost: \"+ str(recall_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Average F1 of XGBoost: \"+ str(f1_score(y_test, y_pred, average='weighted')))\n",
    "print(\"F1 of XGBoost for each type of attack: \"+ str(f1_score(y_test, y_pred, average=None)))\n",
    "xg_f1=f1_score(y_test, y_pred, average=None)\n",
    "cm=confusion_matrix(y_test,y_pred)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "244cad93-747d-49e3-811e-aac3aba44f4c",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "48ece27d-769c-45dc-9be6-9e3737555026",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Found whitespace in feature_names, replace with underlines\n",
      "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001401 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 1851\n",
      "[LightGBM] [Info] Number of data points in the train set: 87791, number of used features: 10\n",
      "[LightGBM] [Info] Start training from score -0.205723\n",
      "[LightGBM] [Info] Start training from score -3.010778\n",
      "[LightGBM] [Info] Start training from score -3.088665\n",
      "[LightGBM] [Info] Start training from score -3.088665\n",
      "[LightGBM] [Info] Start training from score -3.088665\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99     17940\n",
      "           1       1.00      1.00      1.00      1025\n",
      "           2       1.00      0.99      0.99       458\n",
      "           3       0.66      0.29      0.40       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.98     20000\n",
      "   macro avg       0.93      0.85      0.88     20000\n",
      "weighted avg       0.98      0.98      0.98     20000\n",
      "\n",
      "Accuracy of LightGBM: 0.9841\n",
      "Precision of LightGBM: 0.9807666513233602\n",
      "Recall of LightGBM: 0.9841\n",
      "Average F1 of LightGBM: 0.9810852703406319\n",
      "F1 of LightGBM for each type of attack: [0.99118869 1.         0.99340659 0.4        1.        ]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29612 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 7.58 s\n",
      "Wall time: 1.23 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the LightGBM algorithm\n",
    "import lightgbm as lgb\n",
    "lg = lgb.LGBMClassifier()\n",
    "lg.fit(X_train, y_train)\n",
    "y_pred = lg.predict(X_test)\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(\"Accuracy of LightGBM: \"+ str(accuracy_score(y_test, y_pred)))\n",
    "print(\"Precision of LightGBM: \"+ str(precision_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Recall of LightGBM: \"+ str(recall_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Average F1 of LightGBM: \"+ str(f1_score(y_test, y_pred, average='weighted')))\n",
    "print(\"F1 of LightGBM for each type of attack: \"+ str(f1_score(y_test, y_pred, average=None)))\n",
    "lg_f1=f1_score(y_test, y_pred, average=None)\n",
    "cm=confusion_matrix(y_test,y_pred)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfe3225d-44e1-4a23-aa08-3ec0e2bb2bb9",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ecc907f3-923c-4203-aab9-3f157c4566b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99     17940\n",
      "           1       1.00      1.00      1.00      1025\n",
      "           2       1.00      0.98      0.99       458\n",
      "           3       0.67      0.28      0.39       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.98     20000\n",
      "   macro avg       0.93      0.85      0.87     20000\n",
      "weighted avg       0.98      0.98      0.98     20000\n",
      "\n",
      "Accuracy of CatBoost: 0.98395\n",
      "Precision of CatBoost: 0.9805797099769934\n",
      "Recall of CatBoost: 0.98395\n",
      "Average F1 of CatBoost: 0.9807672932567196\n",
      "F1 of CatBoost for each type of attack: [0.99110779 1.         0.99118943 0.390625   0.99767981]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29812 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3min 3s\n",
      "Wall time: 17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the CatBoost algorithm\n",
    "import catboost as cbt\n",
    "cb = cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')\n",
    "cb.fit(X_train, y_train)\n",
    "# \n",
    "y_pred = cb.predict(X_test)\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(\"Accuracy of CatBoost: \"+ str(accuracy_score(y_test, y_pred)))\n",
    "print(\"Precision of CatBoost: \"+ str(precision_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Recall of CatBoost: \"+ str(recall_score(y_test, y_pred, average='weighted')))\n",
    "print(\"Average F1 of CatBoost: \"+ str(f1_score(y_test, y_pred, average='weighted')))\n",
    "print(\"F1 of CatBoost for each type of attack: \"+ str(f1_score(y_test, y_pred, average=None)))\n",
    "cb_f1=f1_score(y_test, y_pred, average=None)\n",
    "cm=confusion_matrix(y_test,y_pred)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3d5851b-98e3-460d-a702-d9acb045786b",
   "metadata": {},
   "source": [
    "## LCCDE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "21fb13b0-b18b-48c9-ae5d-ce652fed589f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Leading model list for each class\n",
    "model=[]\n",
    "for i in range(len(lg_f1)):\n",
    "    if max(lg_f1[i],xg_f1[i],cb_f1[i]) == lg_f1[i]:\n",
    "        model.append(lg)\n",
    "    elif max(lg_f1[i],xg_f1[i],cb_f1[i]) == xg_f1[i]:\n",
    "        model.append(xg)\n",
    "    else:\n",
    "        model.append(cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1c06d21f-b8a2-48dd-a0be-10d666ba8a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "def LCCDE(X_test, y_test, m1, m2, m3):\n",
    "    i = 0\n",
    "    t = []\n",
    "    m = []\n",
    "    yt = []\n",
    "    yp = []\n",
    "    l = []\n",
    "    pred_l = []\n",
    "    pro_l = []\n",
    "\n",
    "   \n",
    "    for xi, yi in stream.iter_pandas(X_test, y_test):\n",
    "\n",
    "        xi2=np.array(list(xi.values()))\n",
    "        y_pred1 = m1.predict(xi2.reshape(1, -1))      # model 1 (LightGBM) makes a prediction on text sample xi\n",
    "        y_pred1 = int(y_pred1[0])\n",
    "        y_pred2 = m2.predict(xi2.reshape(1, -1))      # model 2 (XGBoost) makes a prediction on text sample xi\n",
    "        y_pred2 = int(y_pred2[0])\n",
    "        y_pred3 = m3.predict(xi2.reshape(1, -1))      # model 3 (Catboost) makes a prediction on text sample xi\n",
    "        y_pred3 = int(y_pred3[0])\n",
    "\n",
    "        p1 = m1.predict_proba(xi2.reshape(1, -1))     # The prediction probability (confidence) list of model 1 \n",
    "        p2 = m2.predict_proba(xi2.reshape(1, -1))     # The prediction probability (confidence) list of model 2  \n",
    "        p3 = m3.predict_proba(xi2.reshape(1, -1))     # The prediction probability (confidence) list of model 3  \n",
    "\n",
    "        # Find the highest prediction probability among all classes for each ML model\n",
    "        y_pred_p1 = np.max(p1)\n",
    "        y_pred_p2 = np.max(p2)\n",
    "        y_pred_p3 = np.max(p3)\n",
    "\n",
    "        if y_pred1 == y_pred2 == y_pred3: # If the predicted classes of all the three models are the same\n",
    "            y_pred = y_pred1 # Use this predicted class as the final predicted class\n",
    "\n",
    "        elif y_pred1 != y_pred2 != y_pred3: # If the predicted classes of all the three models are different\n",
    "            # For each prediction model, check if the predicted class’s original ML model is the same as its leader model\n",
    "            if model[y_pred1]==m1: # If they are the same and the leading model is model 1 (LightGBM)\n",
    "                l.append(m1)\n",
    "                pred_l.append(y_pred1) # Save the predicted class\n",
    "                pro_l.append(y_pred_p1) # Save the confidence\n",
    "\n",
    "            if model[y_pred2]==m2: # If they are the same and the leading model is model 2 (XGBoost)\n",
    "                l.append(m2)\n",
    "                pred_l.append(y_pred2)\n",
    "                pro_l.append(y_pred_p2)\n",
    "\n",
    "            if model[y_pred3]==m3: # If they are the same and the leading model is model 3 (CatBoost)\n",
    "                l.append(m3)\n",
    "                pred_l.append(y_pred3)\n",
    "                pro_l.append(y_pred_p3)\n",
    "\n",
    "            if len(l)==0: # Avoid empty probability list\n",
    "                pro_l=[y_pred_p1,y_pred_p2,y_pred_p3]\n",
    "\n",
    "            elif len(l)==1: # If only one pair of the original model and the leader model for each predicted class is the same\n",
    "                y_pred=pred_l[0] # Use the predicted class of the leader model as the final prediction class\n",
    "\n",
    "            else: # If no pair or multiple pairs of the original prediction model and the leader model for each predicted class are the same\n",
    "                max_p = max(pro_l) # Find the highest confidence\n",
    "                \n",
    "                # Use the predicted class with the highest confidence as the final prediction class\n",
    "                if max_p == y_pred_p1:\n",
    "                    y_pred = y_pred1\n",
    "                elif max_p == y_pred_p2:\n",
    "                    y_pred = y_pred2\n",
    "                else:\n",
    "                    y_pred = y_pred3  \n",
    "        \n",
    "        else: # If two predicted classes are the same and the other one is different\n",
    "            n = mode([y_pred1,y_pred2,y_pred3]) # Find the predicted class with the majority vote\n",
    "            y_pred = model[n].predict(xi2.reshape(1, -1)) # Use the predicted class of the leader model as the final prediction class\n",
    "            y_pred = int(y_pred[0]) \n",
    "\n",
    "        yt.append(yi)\n",
    "        yp.append(y_pred) # Save the predicted classes for all tested samples\n",
    "    return yt, yp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1e54ad43-0070-4eba-a742-b171920e977a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 34min 38s\n",
      "Wall time: 4min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Implementing LCCDE\n",
    "yt, yp = LCCDE(X_test, y_test, m1 = lg, m2 = xg, m3 = cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1b77ff8b-50ee-49af-8cca-3d9876d3d1c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of LCCDE: 0.98425\n",
      "Precision of LCCDE: 0.9809127833662222\n",
      "Recall of LCCDE: 0.98425\n",
      "Average F1 of LCCDE: 0.9812349256438232\n",
      "F1 of LCCDE for each type of attack: [0.99127109 1.         0.99671413 0.4        1.        ]\n"
     ]
    }
   ],
   "source": [
    "# The performance of the proposed lCCDE model\n",
    "print(\"Accuracy of LCCDE: \"+ str(accuracy_score(yt, yp)))\n",
    "print(\"Precision of LCCDE: \"+ str(precision_score(yt, yp, average='weighted')))\n",
    "print(\"Recall of LCCDE: \"+ str(recall_score(yt, yp, average='weighted')))\n",
    "print(\"Average F1 of LCCDE: \"+ str(f1_score(yt, yp, average='weighted')))\n",
    "print(\"F1 of LCCDE for each type of attack: \"+ str(f1_score(yt, yp, average=None)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22c797b7-ade8-4b55-96a7-a41becf989b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f144d279-df19-49b9-a5de-0156f008caed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
