{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a11bab8a-922b-4ce7-b8b8-3c40a45d2a97",
   "metadata": {},
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c7b08ce1-1cab-400b-8035-e7266fe04586",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b9f0d06-4a2a-4eea-a2de-275643bd9072",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score,precision_recall_fscore_support\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "import time\n",
    "from river import stream\n",
    "from statistics import mode\n",
    "from statistics import mode\n",
    "from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from statistics import mode\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LassoCV\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, roc_curve, auc, log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import joblib\n",
    "from joblib import load\n",
    "import os\n",
    "from sklearn.preprocessing import PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a500ef73-6ed4-4e58-8d03-a67f0800e45b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/CHDC_FeatureSelect.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7566206-5ccf-41d8-b621-25dc04643086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CAN ID</th>\n",
       "      <th>DLC</th>\n",
       "      <th>DataOne</th>\n",
       "      <th>DataTwo</th>\n",
       "      <th>DataThree</th>\n",
       "      <th>DataFour</th>\n",
       "      <th>DataFive</th>\n",
       "      <th>DataSix</th>\n",
       "      <th>DataSeven</th>\n",
       "      <th>DataEight</th>\n",
       "      <th>Before</th>\n",
       "      <th>Back</th>\n",
       "      <th>Next ID</th>\n",
       "      <th>Pre ID</th>\n",
       "      <th>Next Two ID</th>\n",
       "      <th>Pre Two ID</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1107</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>224</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>909</td>\n",
       "      <td>1042</td>\n",
       "      <td>1056</td>\n",
       "      <td>1102</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1157</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>16</td>\n",
       "      <td>239</td>\n",
       "      <td>1151</td>\n",
       "      <td>1078</td>\n",
       "      <td>339</td>\n",
       "      <td>809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>593</td>\n",
       "      <td>8</td>\n",
       "      <td>201</td>\n",
       "      <td>3</td>\n",
       "      <td>130</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>213</td>\n",
       "      <td>167</td>\n",
       "      <td>126</td>\n",
       "      <td>23</td>\n",
       "      <td>20</td>\n",
       "      <td>688</td>\n",
       "      <td>897</td>\n",
       "      <td>304</td>\n",
       "      <td>544</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1170</td>\n",
       "      <td>8</td>\n",
       "      <td>64</td>\n",
       "      <td>254</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>39</td>\n",
       "      <td>76</td>\n",
       "      <td>1162</td>\n",
       "      <td>902</td>\n",
       "      <td>1151</td>\n",
       "      <td>809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>854</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>128</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>870</td>\n",
       "      <td>1057</td>\n",
       "      <td>871</td>\n",
       "      <td>902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>246</td>\n",
       "      <td>1</td>\n",
       "      <td>133</td>\n",
       "      <td>48</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>871</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>870</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>232</td>\n",
       "      <td>200</td>\n",
       "      <td>108</td>\n",
       "      <td>23</td>\n",
       "      <td>126</td>\n",
       "      <td>183</td>\n",
       "      <td>165</td>\n",
       "      <td>88</td>\n",
       "      <td>25</td>\n",
       "      <td>854</td>\n",
       "      <td>902</td>\n",
       "      <td>870</td>\n",
       "      <td>903</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>871</td>\n",
       "      <td>8</td>\n",
       "      <td>146</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>215</td>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>872</td>\n",
       "      <td>870</td>\n",
       "      <td>1078</td>\n",
       "      <td>854</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>304</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>128</td>\n",
       "      <td>84</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>97</td>\n",
       "      <td>38</td>\n",
       "      <td>24</td>\n",
       "      <td>320</td>\n",
       "      <td>809</td>\n",
       "      <td>897</td>\n",
       "      <td>608</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>1136</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>84</td>\n",
       "      <td>80</td>\n",
       "      <td>84</td>\n",
       "      <td>177</td>\n",
       "      <td>23</td>\n",
       "      <td>18</td>\n",
       "      <td>1107</td>\n",
       "      <td>872</td>\n",
       "      <td>832</td>\n",
       "      <td>871</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CAN ID  DLC  DataOne  DataTwo  DataThree  DataFour  DataFive  DataSix  \\\n",
       "0        1107    5        0      136        141         0       224      256   \n",
       "1        1157    4        3        0          0         0       256      256   \n",
       "2         593    8      201        3        130        76         0      213   \n",
       "3        1170    8       64      254         15         0         0        0   \n",
       "4         854    8        0        0          0       128        19        0   \n",
       "...       ...  ...      ...      ...        ...       ...       ...      ...   \n",
       "99995     832    8        0        0          0        36       246        1   \n",
       "99996     832    8       31      232        200       108        23      126   \n",
       "99997     871    8      146      128          0         0         0        0   \n",
       "99998     304    8        4      128         84       128         0        0   \n",
       "99999    1136    8       21       65          1         4        84       80   \n",
       "\n",
       "       DataSeven  DataEight  Before  Back  Next ID   Pre ID  Next Two ID  \\\n",
       "0            256        256      18    24      909     1042         1056   \n",
       "1            256        256      16   239     1151     1078          339   \n",
       "2            167        126      23    20      688      897          304   \n",
       "3              0        136      39    76     1162      902         1151   \n",
       "4              0          0      25    22      870     1057          871   \n",
       "...          ...        ...     ...   ...      ...      ...          ...   \n",
       "99995        133         48      25    25      871        0            0   \n",
       "99996        183        165      88    25      854      902          870   \n",
       "99997        215         10      24    24      872      870         1078   \n",
       "99998          7         97      38    24      320      809          897   \n",
       "99999         84        177      23    18     1107      872          832   \n",
       "\n",
       "       Pre Two ID  Label  \n",
       "0            1102      0  \n",
       "1             809      0  \n",
       "2             544      0  \n",
       "3             809      0  \n",
       "4             902      0  \n",
       "...           ...    ...  \n",
       "99995         870      0  \n",
       "99996         903      2  \n",
       "99997         854      0  \n",
       "99998         608      0  \n",
       "99999         871      0  \n",
       "\n",
       "[100000 rows x 17 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e46a8255-c352-453e-9417-9c76edf37fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    89407\n",
       "1     5349\n",
       "2     2157\n",
       "3     1919\n",
       "4     1168\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dae95f06-e65e-4e27-a9c5-93967c08169a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(['Label'],axis=1)\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, train_size = 0.8, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "24786bdd-8b24-46ca-8c2b-76b77efd70ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    71467\n",
       "1     4324\n",
       "2     1699\n",
       "3     1557\n",
       "4      953\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "783d53b4-5c72-430f-ac36-1e79bd16954b",
   "metadata": {},
   "source": [
    "0 Normal \\\n",
    "1 DOS\\\n",
    "2 Fuzzing\\\n",
    "3 Replay\\\n",
    "4 Spoofing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f10573c7-c986-43b9-b1f4-db3e77bb2ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote=SMOTE(n_jobs=-1,sampling_strategy={1:10000,2:10000,3:10000,4:10000})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d209155a-1dfb-4d61-8c44-8ff2515e7868",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "693d9ba1-b782-4502-b59d-3ae25ec1faa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    71467\n",
       "4    10000\n",
       "2    10000\n",
       "1    10000\n",
       "3    10000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b41d2827-bccd-441a-aaa1-74c4ca49ad51",
   "metadata": {},
   "source": [
    "# Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6e93ce5-69a9-451e-b34d-4344cf401ac6",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74f28ef8-d571-47e8-96c9-62cb7dafadf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 17.4 s\n",
      "Wall time: 16.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Random Forest training and prediction\n",
    "rf = RandomForestClassifier(random_state = 0)\n",
    "rf.fit(X_train,y_train.astype('int')) \n",
    "rf_score=rf.score(X_test,y_test.astype('int'))\n",
    "y_predict=rf.predict(X_test)\n",
    "y_true=y_test.astype('int')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6f42fb8-a981-4217-ace1-c31385128e22",
   "metadata": {},
   "source": [
    "## ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6b7a5eb9-7f2e-40fd-8246-8a3a7341ad5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 8.45 s\n",
      "Wall time: 8.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "et = ExtraTreesClassifier(random_state = 0)\n",
    "et.fit(X_train,y_train.astype('int')) \n",
    "et_score=et.score(X_test,y_test.astype('int'))\n",
    "y_predict=et.predict(X_test)\n",
    "y_true=y_test.astype('int')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "558a930b-c7c7-434e-a6a9-641730af85e7",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f403fbb-4212-404f-9467-c5795059d746",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 20.3 s\n",
      "Wall time: 1.53 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import xgboost as xgb\n",
    "xg = xgb.XGBClassifier()\n",
    "X_train_x = X_train.values\n",
    "X_test_x = X_test.values\n",
    "xg.fit(X_train_x, y_train)\n",
    "y_pred = xg.predict(X_test_x)\n",
    "\n",
    "xg_f1=f1_score(y_test, y_pred, average=None)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41d3541e-2523-4c3c-bd95-76664dc79276",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f63f0382-8cad-40d4-931c-4ec42eeec125",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Found whitespace in feature_names, replace with underlines\n",
      "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001000 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 3521\n",
      "[LightGBM] [Info] Number of data points in the train set: 111467, number of used features: 16\n",
      "[LightGBM] [Info] Start training from score -0.444493\n",
      "[LightGBM] [Info] Start training from score -2.411143\n",
      "[LightGBM] [Info] Start training from score -2.411143\n",
      "[LightGBM] [Info] Start training from score -2.411143\n",
      "[LightGBM] [Info] Start training from score -2.411143\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "CPU times: total: 11 s\n",
      "Wall time: 1.44 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the LightGBM algorithm\n",
    "import lightgbm as lgb\n",
    "lg = lgb.LGBMClassifier()\n",
    "lg.fit(X_train, y_train)\n",
    "y_pred = lg.predict(X_test)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e68fcd5-69dc-459e-b647-06bc1683aec5",
   "metadata": {},
   "source": [
    "## CATBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "66ae4693-c399-4a77-a3ca-5398d1bfecdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 4min 53s\n",
      "Wall time: 27.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import catboost as cbt\n",
    "cb = cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')\n",
    "cb.fit(X_train, y_train)\n",
    "y_pred = cb.predict(X_test)\n",
    "\n",
    "cb_f1=f1_score(y_test, y_pred, average=None)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "018f5f26-8a2a-4ccc-9589-6d48be0aa5ba",
   "metadata": {},
   "source": [
    "# Featrue Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "95936ee6-549e-4ef9-9954-39ea7990f4ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer = PowerTransformer(method='yeo-johnson')\n",
    "X_scaled = transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6dbbec53-ecb9-4cb7-9cb4-9f4a38e2839d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selected_features: Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "Feature droped： []\n"
     ]
    }
   ],
   "source": [
    "all_features_indices = np.arange(X_scaled.shape[1])\n",
    "lasso = LassoCV(cv=5, random_state=42)\n",
    "lasso.fit(X_scaled, y)\n",
    "selected_features = np.where(lasso.coef_ != 0)[0]\n",
    "print(\"selected_features:\", X.columns[selected_features])\n",
    "discarded_features_name = [X.columns[i] for i in all_features_indices if i not in selected_features]\n",
    "print(\"Feature droped：\", discarded_features_name)\n",
    "X_selected = X_scaled[:, selected_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ba4bc5ed-446b-4d70-b417-247bdc5af83f",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_feature = rf.feature_importances_\n",
    "et_feature = et.feature_importances_\n",
    "xgb_feature = xg.feature_importances_\n",
    "cb_feature = cb.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ce570312-5387-484d-95ce-cec09a98f456",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_feature = rf_feature / rf_feature.max()\n",
    "xgb_feature = xgb_feature / xgb_feature.max()\n",
    "et_feature = et_feature / et_feature.max()\n",
    "cb_feature = cb_feature / cb_feature.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4bfb5c54-79d6-473d-ba64-bde20c131da4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CatBoost Feature Importances: [1.         0.08368678 0.42041672 0.51362171 0.40291395 0.4219112\n",
      " 0.54631193 0.3549212  0.41627677 0.4677244  0.44334923 0.38604245\n",
      " 0.2002459  0.24315041 0.11816585 0.11375542]\n",
      "XGboost Feature Importances: [1.         0.25115559 0.12934689 0.12030954 0.3010828  0.12327426\n",
      " 0.37022704 0.16855313 0.25330985 0.67101276 0.20716946 0.3644421\n",
      " 0.05697363 0.05143566 0.03296527 0.03024095]\n",
      "ET Feature Importances: [1.         0.09675829 0.2926812  0.35307655 0.28857674 0.38120078\n",
      " 0.39621536 0.35861995 0.33231818 0.41066064 0.19187752 0.19132083\n",
      " 0.10570074 0.12740487 0.0622507  0.06575718]\n",
      "RF Feature Importances: [1.         0.05116521 0.18944307 0.26397686 0.2448804  0.21602898\n",
      " 0.30048963 0.19736291 0.25623164 0.3868494  0.26451455 0.3661961\n",
      " 0.0703334  0.10323705 0.04346149 0.04842002]\n"
     ]
    }
   ],
   "source": [
    "print(\"CatBoost Feature Importances:\", cb_feature)\n",
    "print(\"XGboost Feature Importances:\", xgb_feature)\n",
    "print(\"ET Feature Importances:\", et_feature)\n",
    "print(\"RF Feature Importances:\", rf_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b8afa6bd-c517-49f5-bed5-d668e2362bcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average Featrue Imortance: [1.         0.12069147 0.25797197 0.31274616 0.30936347 0.28560381\n",
      " 0.40331099 0.2698643  0.31453411 0.4840618  0.27672769 0.32700037\n",
      " 0.10831342 0.131307   0.06421083 0.06454339]\n"
     ]
    }
   ],
   "source": [
    "avg_feature = (cb_feature  + et_feature + xgb_feature + rf_feature)/4\n",
    "print(\"average Featrue Imortance:\", avg_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "332212ac-0a65-4ef1-b7d2-01c663043373",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features sorted by their score:\n",
      "[(1.0, 'CAN ID'), (0.4841, 'DataEight'), (0.4033, 'DataFive'), (0.327, 'Back'), (0.3145, 'DataSeven'), (0.3127, 'DataTwo'), (0.3094, 'DataThree'), (0.2856, 'DataFour'), (0.2767, 'Before'), (0.2699, 'DataSix'), (0.258, 'DataOne'), (0.1313, ' Pre ID'), (0.1207, 'DLC'), (0.1083, 'Next ID'), (0.0645, 'Pre Two ID'), (0.0642, 'Next Two ID')]\n"
     ]
    }
   ],
   "source": [
    "feature=(df.drop(['Label'],axis=1)).columns.values\n",
    "print (\"Features sorted by their score:\")\n",
    "print (sorted(zip(map(lambda x: round(x, 4), avg_feature), feature), reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c670c64b-fafa-4cae-87b6-c674e9f870f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_feature = rf_feature / rf_feature.max()\n",
    "xgb_feature = xgb_feature / xgb_feature.max()\n",
    "et_feature = et_feature / et_feature.max()\n",
    "cb_feature = cb_feature / cb_feature.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e353b6ab-d7e0-4025-a935-d706a4758c14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected features categories Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "categories number: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 686228 characters]",
      "text/plain": [
       "<Figure size 3000x2400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# \n",
    "plt.rcParams['font.family'] = 'Times New Roman'\n",
    "plt.rcParams['font.size'] = 10  # The font size can be adjusted as needed\n",
    "\n",
    "\n",
    "categories = X.columns[selected_features]\n",
    "print(\"Selected features categories\", categories)\n",
    "N = len(categories)\n",
    "print(\"categories number:\", N)\n",
    "\n",
    "angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()\n",
    "angles += angles[:1]\n",
    "\n",
    "rf_values = rf_feature.tolist()\n",
    "rf_values += rf_values[:1]\n",
    "\n",
    "cb_values = cb_feature.tolist()\n",
    "cb_values += cb_values[:1]\n",
    "\n",
    "et_values = et_feature.tolist()\n",
    "et_values += et_values[:1]\n",
    "\n",
    "xgb_values = xgb_feature.tolist()\n",
    "xgb_values += xgb_values[:1]\n",
    "\n",
    "plt.figure(figsize=(10, 8), dpi=300)\n",
    "ax = plt.subplot(111, polar=True)\n",
    "\n",
    "plt.xticks(angles[:-1], categories, color='black', size=10)\n",
    "ax.plot(angles, rf_values, linewidth=1, linestyle='solid', label='Random Forest')\n",
    "ax.fill(angles, rf_values, 'b', alpha=0.1)\n",
    "ax.plot(angles, et_values, linewidth=1, linestyle='solid', label='ET')\n",
    "ax.fill(angles, et_values, 'r', alpha=0.1)\n",
    "ax.plot(angles, cb_values, linewidth=1, linestyle='solid', label='CatBoost')\n",
    "ax.fill(angles, cb_values, 'r', alpha=0.1)\n",
    "ax.plot(angles, xgb_values, linewidth=1, linestyle='solid', label='XGBoost')\n",
    "ax.fill(angles, xgb_values, 'r', alpha=0.1)\n",
    "ax.spines['polar'].set_visible(False)\n",
    "plt.legend(loc='upper left', bbox_to_anchor=(1, 1), borderaxespad=0, fontsize=10)\n",
    "\n",
    "plt.savefig('feature_importances_comparison.png', dpi=300, bbox_inches='tight')  # save plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4d820a8-ddcc-4fc4-9a9e-2ae636b36868",
   "metadata": {},
   "source": [
    "## RF Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3513c608-54b3-4aae-9397-6a5e8ff57892",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature selected categories Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "categories Number: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 567996 characters]",
      "text/plain": [
       "<Figure size 3000x2400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.family'] = 'Times New Roman'\n",
    "plt.rcParams['font.size'] = 10  # \n",
    "rf = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "rf_importances = rf.feature_importances_\n",
    "\n",
    "\n",
    "# Normalize the importance of features\n",
    "rf_importances = rf_importances / rf_importances.max()\n",
    "\n",
    "\n",
    "# Create a radar chart\n",
    "categories = X.columns[selected_features]\n",
    "print(\"Feature selected categories\", categories)\n",
    "N = len(categories)\n",
    "print(\"categories Number:\", N)\n",
    "\n",
    "angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()\n",
    "angles += angles[:1]\n",
    "\n",
    "rf_values = rf_importances.tolist()\n",
    "rf_values += rf_values[:1]\n",
    "\n",
    "plt.figure(figsize=(10, 8), dpi=300)\n",
    "ax = plt.subplot(111, polar=True)\n",
    "\n",
    "plt.xticks(angles[:-1], categories, color='black', size=10)\n",
    "ax.plot(angles, rf_values, linewidth=1, linestyle='solid', color='b',label='Random Forest')\n",
    "ax.fill(angles, rf_values, 'b', alpha=0.1)\n",
    "\n",
    "ax.spines['polar'].set_visible(False)\n",
    "\n",
    "# plt.legend(loc='upper right')\n",
    "# plt.legend(loc='upper left', bbox_to_anchor=(0.8, 0.0), borderaxespad=0)  \n",
    "plt.legend(loc='upper left', bbox_to_anchor=(1, 1), borderaxespad=0, fontsize=10)\n",
    "\n",
    "plt.savefig('feature_importances_RF.png', dpi=300, bbox_inches='tight')  # Save plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56a2a218-25b3-484c-99d8-a01e9d70e268",
   "metadata": {},
   "source": [
    "## ET Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a4622b1d-d886-46b0-9fb6-bc1150e5ebdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature selected categories Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "categories number: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 561228 characters]",
      "text/plain": [
       "<Figure size 3000x2400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.family'] = 'Times New Roman'\n",
    "plt.rcParams['font.size'] = 10  # \n",
    "et = ExtraTreesClassifier(random_state = 0)\n",
    "et.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "et_importances = et.feature_importances_\n",
    "\n",
    "\n",
    "\n",
    "et_importances = et_importances / et_importances.max()\n",
    "\n",
    "\n",
    "\n",
    "categories = X.columns[selected_features]\n",
    "print(\"Feature selected categories\", categories)\n",
    "N = len(categories)\n",
    "print(\"categories number:\", N)\n",
    "\n",
    "angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()\n",
    "angles += angles[:1]\n",
    "\n",
    "et_values = et_importances.tolist()\n",
    "et_values += et_values[:1]\n",
    "\n",
    "plt.figure(figsize=(10, 8), dpi=300)\n",
    "ax = plt.subplot(111, polar=True)\n",
    "\n",
    "plt.xticks(angles[:-1], categories, color='black', size=10)\n",
    "ax.plot(angles, et_values, linewidth=1, linestyle='solid', color='orange',label='Extra Tree')\n",
    "ax.fill(angles, et_values, 'r', alpha=0.1)\n",
    "\n",
    "ax.spines['polar'].set_visible(False)\n",
    "\n",
    "\n",
    "plt.legend(loc='upper left', bbox_to_anchor=(1, 1), borderaxespad=0, fontsize=10)\n",
    "\n",
    "plt.savefig('feature_importances_ET.png', dpi=300, bbox_inches='tight')  # save plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3cb872e-dd26-4ad3-af8c-f3067df7511d",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "17ac3996-49b1-4c19-9f10-0a20caadecd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature selected categories Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "categories number: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 580828 characters]",
      "text/plain": [
       "<Figure size 3000x2400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.family'] = 'Times New Roman'\n",
    "plt.rcParams['font.size'] = 10  # \n",
    "cb = cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')\n",
    "cb.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "cb_importances = cb.feature_importances_\n",
    "\n",
    "\n",
    "\n",
    "cb_importances = cb_importances / cb_importances.max()\n",
    "\n",
    "\n",
    "\n",
    "categories = X.columns[selected_features]\n",
    "print(\"Feature selected categories\", categories)\n",
    "N = len(categories)\n",
    "print(\"categories number:\", N)\n",
    "\n",
    "angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()\n",
    "angles += angles[:1]\n",
    "\n",
    "cb_values = cb_importances.tolist()\n",
    "cb_values += cb_values[:1]\n",
    "\n",
    "plt.figure(figsize=(10, 8), dpi=300)\n",
    "ax = plt.subplot(111, polar=True)\n",
    "\n",
    "plt.xticks(angles[:-1], categories, color='black', size=10)\n",
    "ax.plot(angles, cb_values, linewidth=1, linestyle='solid', color='green',label='CatBoost')\n",
    "ax.fill(angles, cb_values, 'r', alpha=0.1)\n",
    "\n",
    "ax.spines['polar'].set_visible(False)\n",
    "\n",
    "plt.legend(loc='upper left', bbox_to_anchor=(1, 1), borderaxespad=0, fontsize=10)\n",
    "\n",
    "plt.savefig('feature_importances_CatBoost.png', dpi=300, bbox_inches='tight')  # Save Plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ab676e2-4df8-4a40-ad32-5de9bf2634d9",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ecef6c0b-d7e4-4b60-b69a-364b5652d19c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature selected categories Index(['CAN ID', 'DLC', 'DataOne', 'DataTwo', 'DataThree', 'DataFour',\n",
      "       'DataFive', 'DataSix', 'DataSeven', 'DataEight', 'Before', 'Back',\n",
      "       'Next ID', ' Pre ID', 'Next Two ID', 'Pre Two ID'],\n",
      "      dtype='object')\n",
      "categories number: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 567016 characters]",
      "text/plain": [
       "<Figure size 3000x2400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.family'] = 'Times New Roman'\n",
    "plt.rcParams['font.size'] = 10  # \n",
    "xg = xgb.XGBClassifier()\n",
    "xg.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "xg_importances = xg.feature_importances_\n",
    "\n",
    "\n",
    "\n",
    "xg_importances = xg_importances / xg_importances.max()\n",
    "\n",
    "\n",
    "\n",
    "categories = X.columns[selected_features]\n",
    "print(\"Feature selected categories\", categories)\n",
    "N = len(categories)\n",
    "print(\"categories number:\", N)\n",
    "\n",
    "angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()\n",
    "angles += angles[:1]\n",
    "\n",
    "xg_values = xg_importances.tolist()\n",
    "xg_values += xg_values[:1]\n",
    "\n",
    "plt.figure(figsize=(10, 8), dpi=300)\n",
    "ax = plt.subplot(111, polar=True)\n",
    "\n",
    "plt.xticks(angles[:-1], categories, color='black', size=10)\n",
    "ax.plot(angles, xg_values, linewidth=1, linestyle='solid', color='r',label='XGBoost')\n",
    "ax.fill(angles, xg_values, 'r', alpha=0.1)\n",
    "\n",
    "ax.spines['polar'].set_visible(False)\n",
    "\n",
    "plt.legend(loc='upper left', bbox_to_anchor=(1, 1), borderaxespad=0, fontsize=10)\n",
    "\n",
    "plt.savefig('feature_importances_XGBoost.png', dpi=300, bbox_inches='tight')  # Save plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c3ff018-1eef-451e-8081-0332c229951d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
