{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a11bab8a-922b-4ce7-b8b8-3c40a45d2a97",
   "metadata": {},
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c7b08ce1-1cab-400b-8035-e7266fe04586",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b9f0d06-4a2a-4eea-a2de-275643bd9072",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score,precision_recall_fscore_support\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "import time\n",
    "from river import stream\n",
    "from statistics import mode\n",
    "from statistics import mode\n",
    "from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from statistics import mode\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LassoCV\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, roc_curve, auc, log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import joblib\n",
    "from joblib import load\n",
    "import os\n",
    "from sklearn.preprocessing import PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a500ef73-6ed4-4e58-8d03-a67f0800e45b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/CHDC_FeatureSelect.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7566206-5ccf-41d8-b621-25dc04643086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CAN ID</th>\n",
       "      <th>DLC</th>\n",
       "      <th>DataOne</th>\n",
       "      <th>DataTwo</th>\n",
       "      <th>DataThree</th>\n",
       "      <th>DataFour</th>\n",
       "      <th>DataFive</th>\n",
       "      <th>DataSix</th>\n",
       "      <th>DataSeven</th>\n",
       "      <th>DataEight</th>\n",
       "      <th>Before</th>\n",
       "      <th>Back</th>\n",
       "      <th>Next ID</th>\n",
       "      <th>Pre ID</th>\n",
       "      <th>Next Two ID</th>\n",
       "      <th>Pre Two ID</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1107</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>224</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>909</td>\n",
       "      <td>1042</td>\n",
       "      <td>1056</td>\n",
       "      <td>1102</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1157</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>16</td>\n",
       "      <td>239</td>\n",
       "      <td>1151</td>\n",
       "      <td>1078</td>\n",
       "      <td>339</td>\n",
       "      <td>809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>593</td>\n",
       "      <td>8</td>\n",
       "      <td>201</td>\n",
       "      <td>3</td>\n",
       "      <td>130</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>213</td>\n",
       "      <td>167</td>\n",
       "      <td>126</td>\n",
       "      <td>23</td>\n",
       "      <td>20</td>\n",
       "      <td>688</td>\n",
       "      <td>897</td>\n",
       "      <td>304</td>\n",
       "      <td>544</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1170</td>\n",
       "      <td>8</td>\n",
       "      <td>64</td>\n",
       "      <td>254</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "      <td>39</td>\n",
       "      <td>76</td>\n",
       "      <td>1162</td>\n",
       "      <td>902</td>\n",
       "      <td>1151</td>\n",
       "      <td>809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>854</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>128</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>870</td>\n",
       "      <td>1057</td>\n",
       "      <td>871</td>\n",
       "      <td>902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>246</td>\n",
       "      <td>1</td>\n",
       "      <td>133</td>\n",
       "      <td>48</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>871</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>870</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>832</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>232</td>\n",
       "      <td>200</td>\n",
       "      <td>108</td>\n",
       "      <td>23</td>\n",
       "      <td>126</td>\n",
       "      <td>183</td>\n",
       "      <td>165</td>\n",
       "      <td>88</td>\n",
       "      <td>25</td>\n",
       "      <td>854</td>\n",
       "      <td>902</td>\n",
       "      <td>870</td>\n",
       "      <td>903</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>871</td>\n",
       "      <td>8</td>\n",
       "      <td>146</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>215</td>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>872</td>\n",
       "      <td>870</td>\n",
       "      <td>1078</td>\n",
       "      <td>854</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>304</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>128</td>\n",
       "      <td>84</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>97</td>\n",
       "      <td>38</td>\n",
       "      <td>24</td>\n",
       "      <td>320</td>\n",
       "      <td>809</td>\n",
       "      <td>897</td>\n",
       "      <td>608</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>1136</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>84</td>\n",
       "      <td>80</td>\n",
       "      <td>84</td>\n",
       "      <td>177</td>\n",
       "      <td>23</td>\n",
       "      <td>18</td>\n",
       "      <td>1107</td>\n",
       "      <td>872</td>\n",
       "      <td>832</td>\n",
       "      <td>871</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CAN ID  DLC  DataOne  DataTwo  DataThree  DataFour  DataFive  DataSix  \\\n",
       "0        1107    5        0      136        141         0       224      256   \n",
       "1        1157    4        3        0          0         0       256      256   \n",
       "2         593    8      201        3        130        76         0      213   \n",
       "3        1170    8       64      254         15         0         0        0   \n",
       "4         854    8        0        0          0       128        19        0   \n",
       "...       ...  ...      ...      ...        ...       ...       ...      ...   \n",
       "99995     832    8        0        0          0        36       246        1   \n",
       "99996     832    8       31      232        200       108        23      126   \n",
       "99997     871    8      146      128          0         0         0        0   \n",
       "99998     304    8        4      128         84       128         0        0   \n",
       "99999    1136    8       21       65          1         4        84       80   \n",
       "\n",
       "       DataSeven  DataEight  Before  Back  Next ID   Pre ID  Next Two ID  \\\n",
       "0            256        256      18    24      909     1042         1056   \n",
       "1            256        256      16   239     1151     1078          339   \n",
       "2            167        126      23    20      688      897          304   \n",
       "3              0        136      39    76     1162      902         1151   \n",
       "4              0          0      25    22      870     1057          871   \n",
       "...          ...        ...     ...   ...      ...      ...          ...   \n",
       "99995        133         48      25    25      871        0            0   \n",
       "99996        183        165      88    25      854      902          870   \n",
       "99997        215         10      24    24      872      870         1078   \n",
       "99998          7         97      38    24      320      809          897   \n",
       "99999         84        177      23    18     1107      872          832   \n",
       "\n",
       "       Pre Two ID  Label  \n",
       "0            1102      0  \n",
       "1             809      0  \n",
       "2             544      0  \n",
       "3             809      0  \n",
       "4             902      0  \n",
       "...           ...    ...  \n",
       "99995         870      0  \n",
       "99996         903      2  \n",
       "99997         854      0  \n",
       "99998         608      0  \n",
       "99999         871      0  \n",
       "\n",
       "[100000 rows x 17 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e46a8255-c352-453e-9417-9c76edf37fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    89407\n",
       "1     5349\n",
       "2     2157\n",
       "3     1919\n",
       "4     1168\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dae95f06-e65e-4e27-a9c5-93967c08169a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(['Label'],axis=1)\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, train_size = 0.8, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "24786bdd-8b24-46ca-8c2b-76b77efd70ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    71467\n",
       "1     4324\n",
       "2     1699\n",
       "3     1557\n",
       "4      953\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "783d53b4-5c72-430f-ac36-1e79bd16954b",
   "metadata": {},
   "source": [
    "0 Normal \\\n",
    "1 DOS\\\n",
    "2 Fuzzing\\\n",
    "3 Replay\\\n",
    "4 Spoofing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f10573c7-c986-43b9-b1f4-db3e77bb2ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote=SMOTE(n_jobs=-1,sampling_strategy={2:4000,3:4000,4:4000})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d209155a-1dfb-4d61-8c44-8ff2515e7868",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "693d9ba1-b782-4502-b59d-3ae25ec1faa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    71467\n",
       "1     4324\n",
       "4     4000\n",
       "2     4000\n",
       "3     4000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b41d2827-bccd-441a-aaa1-74c4ca49ad51",
   "metadata": {},
   "source": [
    "# Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6e93ce5-69a9-451e-b34d-4344cf401ac6",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74f28ef8-d571-47e8-96c9-62cb7dafadf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 11.2 s\n",
      "Wall time: 11.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Random Forest training and prediction\n",
    "rf = RandomForestClassifier(random_state = 0)\n",
    "rf.fit(X_train,y_train.astype('int')) \n",
    "rf_score=rf.score(X_test,y_test.astype('int'))\n",
    "y_predict=rf.predict(X_test)\n",
    "y_true=y_test.astype('int')\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6f42fb8-a981-4217-ace1-c31385128e22",
   "metadata": {},
   "source": [
    "## ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6b7a5eb9-7f2e-40fd-8246-8a3a7341ad5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 5.92 s\n",
      "Wall time: 5.94 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Extra trees training and prediction\n",
    "et = ExtraTreesClassifier(random_state = 0)\n",
    "et.fit(X_train,y_train.astype('int')) \n",
    "et_score=et.score(X_test,y_test.astype('int'))\n",
    "y_predict=et.predict(X_test)\n",
    "y_true=y_test.astype('int')\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "558a930b-c7c7-434e-a6a9-641730af85e7",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f403fbb-4212-404f-9467-c5795059d746",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 15.5 s\n",
      "Wall time: 1.08 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the XGBoost algorithm\n",
    "import xgboost as xgb\n",
    "xg = xgb.XGBClassifier()\n",
    "X_train_x = X_train.values\n",
    "X_test_x = X_test.values\n",
    "xg.fit(X_train_x, y_train)\n",
    "y_pred = xg.predict(X_test_x)\n",
    "xg_f1=f1_score(y_test, y_pred, average=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e68fcd5-69dc-459e-b647-06bc1683aec5",
   "metadata": {},
   "source": [
    "## CATBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "66ae4693-c399-4a77-a3ca-5398d1bfecdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 4min 1s\n",
      "Wall time: 21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the CatBoost algorithm\n",
    "import catboost as cbt\n",
    "cb = cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')\n",
    "cb.fit(X_train, y_train)\n",
    "y_pred = cb.predict(X_test)\n",
    "cb_f1=f1_score(y_test, y_pred, average=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a43a19b-787e-458c-8b9e-56b5ed8f216e",
   "metadata": {},
   "source": [
    "## Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "439d9134-9c47-4924-bc19-4ae8b03941cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3min 9s\n",
      "Wall time: 1min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "base_classifiers = [\n",
    "    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "    ('et', ExtraTreesClassifier(n_estimators=100, random_state=42)),\n",
    "    ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "]\n",
    "\n",
    "meta_classifier = cbt.CatBoostClassifier(n_estimators=100, random_state=42, verbose=0)\n",
    "\n",
    "stacking_clf = StackingClassifier(estimators=base_classifiers, final_estimator=meta_classifier)\n",
    "\n",
    "stacking_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = stacking_clf.predict(X_test)\n",
    "\n",
    "stacking_f1 = f1_score(y_test, y_pred, average=None)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "26aa4e1b-073c-42d9-88d0-bfc2c32d968e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimated Total FLOPs for Stacking Classifier: 7023.28 MFLOPs\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, StackingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "import xgboost as xgb\n",
    "from catboost import CatBoostClassifier\n",
    "\n",
    "\n",
    "stacking_f1 = f1_score(y_test, y_pred, average=None)\n",
    "\n",
    "# Function to estimate FLOPs\n",
    "def estimate_flops_rf(n_samples, n_features, n_estimators, max_depth):\n",
    "    # Assuming each tree in RF has O(n_samples * n_features * max_depth) FLOPs\n",
    "    return n_samples * n_features * n_estimators * max_depth\n",
    "\n",
    "def estimate_flops_et(n_samples, n_features, n_estimators, max_depth):\n",
    "    # Similar to RF, but ET may have a different max_depth\n",
    "    return n_samples * n_features * n_estimators * max_depth\n",
    "\n",
    "def estimate_flops_xgb(n_samples, n_features, n_estimators, max_depth):\n",
    "    # XGBoost is more complex; includes gradient boosting steps\n",
    "    # Estimate FLOPs higher than RF/ET\n",
    "    return n_samples * n_features * n_estimators * max_depth * 2\n",
    "\n",
    "def estimate_flopsCatBoost(n_samples, n_features, iterations, depth):\n",
    "    # CatBoost parameters\n",
    "    return n_samples * n_features * iterations * depth * 3\n",
    "\n",
    "# Get input data shape\n",
    "n_samples, n_features = X_train.shape\n",
    "\n",
    "# Assume max_depth for each base classifier\n",
    "max_depth_rf = 10\n",
    "max_depth_et = 10\n",
    "max_depth_xgb = 6\n",
    "\n",
    "# Estimate FLOPs for each base classifier\n",
    "rf_flops = estimate_flops_rf(n_samples, n_features, 100, max_depth_rf)\n",
    "et_flops = estimate_flops_et(n_samples, n_features, 100, max_depth_et)\n",
    "xgb_flops = estimate_flops_xgb(n_samples, n_features, 100, max_depth_xgb)\n",
    "\n",
    "# Estimate FLOPs for the meta-classifier (CatBoost)\n",
    "cb_flops = estimate_flopsCatBoost(n_samples, n_features, 100, 6)\n",
    "\n",
    "# Total FLOPs (sum of base classifiers and meta-classifier)\n",
    "total_flops = rf_flops + et_flops + xgb_flops + cb_flops\n",
    "\n",
    "# Convert to MFLOPs\n",
    "total_mflops = total_flops / 1e6\n",
    "\n",
    "print(f\"Estimated Total FLOPs for Stacking Classifier: {total_mflops:.2f} MFLOPs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4df4d6d-13bc-48c7-9ee9-2ff363f3912e",
   "metadata": {},
   "source": [
    "## BLending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6c35de35-8a2d-49a4-876b-13fc73d23895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 4min 27s\n",
      "Wall time: 33.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "\n",
    "base_classifiers = [\n",
    "    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "    ('cb', cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')),\n",
    "    ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "]\n",
    "\n",
    "\n",
    "blending_clf = VotingClassifier(estimators=base_classifiers, weights=None, voting='soft')\n",
    "\n",
    "\n",
    "blending_clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = blending_clf.predict(X_test)\n",
    "\n",
    "\n",
    "blending_f1 = f1_score(y_test, y_pred, average=None)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d3606275-7e7f-4270-ba55-044320e985ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: [0.9958331  0.99951243 0.98342541 0.75       1.        ]\n",
      "FLOPs for Random Forest: 1404.66 MFLOPs\n",
      "FLOPs for CatBoost: 1404.66 MFLOPs\n",
      "FLOPs for XGBoost: 1404.66 MFLOPs\n",
      "Total estimated FLOPs for blending_clf: 4213.97 MFLOPs\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "\n",
    "\n",
    "base_classifiers = [\n",
    "    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "    ('cb', cbt.CatBoostClassifier(verbose=0, boosting_type='Plain', iterations=100)),  #  iterations\n",
    "    ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', n_estimators=100))  # n_estimators\n",
    "]\n",
    "\n",
    "#  VotingClassifier\n",
    "blending_clf = VotingClassifier(estimators=base_classifiers, voting='soft')\n",
    "\n",
    "# \n",
    "# X_train = ...\n",
    "# y_train = ...\n",
    "# X_test = ...\n",
    "# y_test = ...\n",
    "\n",
    "# \n",
    "blending_clf.fit(X_train, y_train)\n",
    "\n",
    "# \n",
    "y_pred = blending_clf.predict(X_test)\n",
    "\n",
    "# \n",
    "blending_f1 = f1_score(y_test, y_pred, average=None)\n",
    "print(\"F1 Score:\", blending_f1)\n",
    "\n",
    "# \n",
    "def estimate_flops_rf(n_samples, n_features, n_estimators, max_depth):\n",
    "    \"\"\"\n",
    "    \n",
    "    \"\"\"\n",
    "    return n_samples * n_features * n_estimators * max_depth\n",
    "\n",
    "def estimate_flops_tree_based(n_samples, n_features, n_trees, max_depth):\n",
    "    \"\"\"\n",
    "    \n",
    "    \"\"\"\n",
    "    return n_samples * n_features * n_trees * max_depth\n",
    "\n",
    "\n",
    "n_samples, n_features = X_train.shape\n",
    "\n",
    "\n",
    "rf_flops = estimate_flops_rf(n_samples, n_features, base_classifiers[0][1].n_estimators, 10)  \n",
    "print(f\"FLOPs for Random Forest: {rf_flops / 1e6:.2f} MFLOPs\")\n",
    "\n",
    "\n",
    "cb_iterations = base_classifiers[1][1].get_params()['iterations']\n",
    "cb_flops = estimate_flops_tree_based(n_samples, n_features, cb_iterations, 10)  \n",
    "print(f\"FLOPs for CatBoost: {cb_flops / 1e6:.2f} MFLOPs\")\n",
    "\n",
    "\n",
    "xg_flops = estimate_flops_tree_based(n_samples, n_features, base_classifiers[2][1].n_estimators, 10)  \n",
    "print(f\"FLOPs for XGBoost: {xg_flops / 1e6:.2f} MFLOPs\")\n",
    "\n",
    "\n",
    "total_flops = rf_flops + cb_flops + xg_flops\n",
    "print(f\"Total estimated FLOPs for blending_clf: {total_flops / 1e6:.2f} MFLOPs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e23f7a8d-063f-4e09-ad79-7829cb29221f",
   "metadata": {},
   "source": [
    "# BEPCD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "292fb011-b9a1-44f3-9517-b958556368aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=[]\n",
    "for i in range(len(blending_f1)):\n",
    "    if max(blending_f1[i],stacking_f1[i]) == stacking_f1[i]:\n",
    "        model.append(stacking_clf)\n",
    "    else:\n",
    "        model.append(blending_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3b4f55fe-7ac4-432a-8312-9b02a972e856",
   "metadata": {},
   "outputs": [],
   "source": [
    "def BEPCD(X_test, y_test, m1, m2,):\n",
    "    i = 0\n",
    "    t = []\n",
    "    m = []\n",
    "    yt = []\n",
    "    yp = []\n",
    "    l = []\n",
    "    pred_l = []\n",
    "    pro_l = []\n",
    "\n",
    "    # For each class (normal or a type of attack), find the pioneer model\n",
    "    for xi, yi in stream.iter_pandas(X_test, y_test):\n",
    "\n",
    "        xi2=np.array(list(xi.values()))\n",
    "        y_pred1 = m1.predict(xi2.reshape(1, -1))      \n",
    "        y_pred1 = int(y_pred1[0])\n",
    "        y_pred2 = m2.predict(xi2.reshape(1, -1))      \n",
    "        y_pred2 = int(y_pred2[0])\n",
    "\n",
    "\n",
    "        p1 = m1.predict_proba(xi2.reshape(1, -1))      \n",
    "        p2 = m2.predict_proba(xi2.reshape(1, -1))    \n",
    "\n",
    "\n",
    "        \n",
    "        y_pred_p1 = np.max(p1)\n",
    "        y_pred_p2 = np.max(p2)\n",
    "\n",
    "        if y_pred1 == y_pred2: \n",
    "            y_pred = y_pred1 \n",
    "\n",
    "        elif y_pred1 != y_pred2:\n",
    "            # For each prediction model, check if the predicted class’s original ML model is the same as its pioneer model\n",
    "            if model[y_pred1]==m1 and model[y_pred2]!=m2: \n",
    "                l.append(m1)\n",
    "                pred_l.append(y_pred1) \n",
    "                pro_l.append(y_pred_p1) \n",
    "\n",
    "            if model[y_pred2]==m2 and model[y_pred1]!=m1: \n",
    "                l.append(m2)\n",
    "                pred_l.append(y_pred2)\n",
    "                pro_l.append(y_pred_p2)\n",
    "\n",
    "            if len(l)==0: \n",
    "                pro_l=[y_pred_p1,y_pred_p2]              \n",
    "            else:\n",
    "                max_p = max(pro_l)\n",
    "                \n",
    "                if max_p == y_pred_p1:\n",
    "                    y_pred = y_pred1\n",
    "                else:\n",
    "                    y_pred = y_pred2  \n",
    "        \n",
    "\n",
    "        yt.append(yi)\n",
    "        yp.append(y_pred) \n",
    "    return yt, yp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ca7dd83a-1ff2-4f00-9562-ed1f8fde4c98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "====================  1 Folding verification ====================\n",
      "Training set size: 87844, Verification set size: 20000\n",
      "\n",
      "The current discounted assessment result:\n",
      "Accuracy: 0.9913000000000000\n",
      "Presision: 0.9911471242962220\n",
      "Recall: 0.9913000000000000\n",
      "F1: 0.9904753327609469\n",
      "\n",
      "F1 scores of various categories:\n",
      "类别 0: 0.9953646822294203\n",
      "类别 1: 0.9981549815498155\n",
      "类别 2: 0.9760000000000001\n",
      "类别 3: 0.7621951219512196\n",
      "类别 4: 0.9979550102249488\n",
      "\n",
      "Confusion matrix:\n",
      "[[17823     1     2     4     0]\n",
      " [    0  1082     0     0     0]\n",
      " [   15     0   427     4     0]\n",
      " [  144     3     0   250     1]\n",
      " [    0     0     0     0   244]]\n",
      "\n",
      "====================  2 Folding verification ====================\n",
      "Training set size: 87758, Verification set size: 20000\n",
      "\n",
      "The current discounted assessment result:\n",
      "Accuracy: 0.9931000000000000\n",
      "Presision: 0.9929699422343010\n",
      "Recall: 0.9931000000000000\n",
      "F1: 0.9925700572514615\n",
      "\n",
      "F1 scores of various categories:\n",
      "类别 0: 0.9962957805319593\n",
      "类别 1: 0.9977324263038548\n",
      "类别 2: 0.9875621890547264\n",
      "类别 3: 0.8006430868167201\n",
      "类别 4: 0.9913793103448276\n",
      "\n",
      "Confusion matrix:\n",
      "[[17886     3     1     6     2]\n",
      " [    0  1100     0     0     0]\n",
      " [    7     0   397     2     0]\n",
      " [  113     2     0   249     1]\n",
      " [    1     0     0     0   230]]\n",
      "\n",
      "====================  3 Folding verification ====================\n",
      "Training set size: 87795, Verification set size: 20000\n",
      "\n",
      "The current discounted assessment result:\n",
      "Accuracy: 0.9930500000000000\n",
      "Presision: 0.9928727067203541\n",
      "Recall: 0.9930500000000000\n",
      "F1: 0.9924944292037015\n",
      "\n",
      "F1 scores of various categories:\n",
      "类别 0: 0.9963210702341138\n",
      "类别 1: 0.9990723562152133\n",
      "类别 2: 0.9837209302325581\n",
      "类别 3: 0.7915309446254072\n",
      "类别 4: 0.9959183673469388\n",
      "\n",
      "Confusion matrix:\n",
      "[[17874     2     1     7     0]\n",
      " [    0  1077     0     0     0]\n",
      " [    7     0   423     3     1]\n",
      " [  115     0     2   243     1]\n",
      " [    0     0     0     0   244]]\n",
      "\n",
      "====================  4 Folding verification ====================\n",
      "Training set size: 87781, Verification set size: 20000\n",
      "\n",
      "The current discounted assessment result:\n",
      "Accuracy: 0.9915000000000000\n",
      "Presision: 0.9912050565046360\n",
      "Recall: 0.9915000000000000\n",
      "F1: 0.9907499048474634\n",
      "\n",
      "F1 scores of various categories:\n",
      "类别 0: 0.9956169551708833\n",
      "类别 1: 0.9965703086722195\n",
      "类别 2: 0.9758342922899885\n",
      "类别 3: 0.7564102564102564\n",
      "类别 4: 0.9952153110047847\n",
      "\n",
      "Confusion matrix:\n",
      "[[17945     3     1     9     0]\n",
      " [    0  1017     0     0     0]\n",
      " [   14     0   424     5     0]\n",
      " [  131     4     1   236     2]\n",
      " [    0     0     0     0   208]]\n",
      "\n",
      "====================  5 Folding verification ====================\n",
      "Training set size: 87846, Verification set size: 20000\n",
      "\n",
      "The current discounted assessment result:\n",
      "Accuracy: 0.9913500000000000\n",
      "Presision: 0.9911015183224983\n",
      "Recall: 0.9913500000000000\n",
      "F1: 0.9904994922027863\n",
      "\n",
      "F1 scores of various categories:\n",
      "类别 0: 0.9953654587079123\n",
      "类别 1: 0.9976754997675499\n",
      "类别 2: 0.9929411764705882\n",
      "类别 3: 0.7593123209169056\n",
      "类别 4: 0.9979296066252588\n",
      "\n",
      "Confusion matrix:\n",
      "[[17826     1     0     9     1]\n",
      " [    0  1073     0     0     0]\n",
      " [    2     1   422     3     0]\n",
      " [  153     3     0   265     0]\n",
      " [    0     0     0     0   241]]\n",
      "\n",
      "==================== Summary of 5-fold cross-validation ====================\n",
      "Accuracy: 0.9920600000000001 ± 0.0008315046602395\n",
      "Precision: 0.9918592696156022 ± 0.0008683295692180\n",
      "Recall: 0.9920600000000001 ± 0.0008315046602395\n",
      "F1: 0.9913578432532718 ± 0.0009639990456750\n",
      "\n",
      "Average F1 scores of all categories:\n",
      "CLASS 0: 0.9957927893748579 ± 0.0004310187152671\n",
      "CLASS 1: 0.9978411145017306 ± 0.0008085366413154\n",
      "CLASS 2: 0.9832117176095723 ± 0.0066375227946119\n",
      "CLASS 3: 0.7740183461441018 ± 0.0183394032281579\n",
      "CLASS 4: 0.9956795211093518 ± 0.0024085301206261\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Initialize and store the list of each folding result\n",
    "accuracy_scores = []\n",
    "precision_scores = []\n",
    "recall_scores = []\n",
    "f1_scores = []\n",
    "f1_scores_per_class = []\n",
    "\n",
    "# Create a 5-fold cross-validation\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "\n",
    "for fold, (train_index, val_index) in enumerate(kf.split(X), 1):\n",
    "    print(f\"\\n{'='*20}  {fold} Folding verification {'='*20}\")\n",
    "    \n",
    "     \n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "    \n",
    "    # SMOTE\n",
    "    X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))\n",
    "    \n",
    "    print(f\"Training set size: {X_train.shape[0]}, Verification set size: {X_val.shape[0]}\")\n",
    "    \n",
    "    # \n",
    "    base_classifiers = [\n",
    "        ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "        ('cb', cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')),\n",
    "        ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "    ]\n",
    "    blending_clf = VotingClassifier(estimators=base_classifiers, weights=None, voting='soft')\n",
    "    blending_clf.fit(X_train, y_train)\n",
    "    \n",
    "    # \n",
    "    base_classifiers_stack = [\n",
    "        ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "        ('et', ExtraTreesClassifier(n_estimators=100, random_state=42)),\n",
    "        ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "    ]\n",
    "    meta_classifier = cbt.CatBoostClassifier(n_estimators=100, random_state=42, verbose=0)\n",
    "    stacking_clf = StackingClassifier(estimators=base_classifiers_stack, final_estimator=meta_classifier)\n",
    "    stacking_clf.fit(X_train, y_train)\n",
    "    \n",
    "    # \n",
    "    yt, yp = BEPCD(X_val, y_val, m1=blending_clf, m2=stacking_clf)\n",
    "    \n",
    "    # \n",
    "    current_accuracy = accuracy_score(yt, yp)\n",
    "    current_precision = precision_score(yt, yp, average='weighted')\n",
    "    current_recall = recall_score(yt, yp, average='weighted')\n",
    "    current_f1 = f1_score(yt, yp, average='weighted')\n",
    "    current_f1_per_class = f1_score(yt, yp, average=None)\n",
    "    \n",
    "    # \n",
    "    accuracy_scores.append(current_accuracy)\n",
    "    precision_scores.append(current_precision)\n",
    "    recall_scores.append(current_recall)\n",
    "    f1_scores.append(current_f1)\n",
    "    f1_scores_per_class.append(current_f1_per_class)\n",
    "    \n",
    "    # \n",
    "    print(f\"\\nThe current discounted assessment result:\")\n",
    "    print(f\"Accuracy: {current_accuracy:.16f}\")\n",
    "    print(f\"Presision: {current_precision:.16f}\")\n",
    "    print(f\"Recall: {current_recall:.16f}\")\n",
    "    print(f\"F1: {current_f1:.16f}\")\n",
    "    \n",
    "    print(\"\\nF1 scores of various categories:\")\n",
    "    for i, f1 in enumerate(current_f1_per_class):\n",
    "        print(f\"CLASS {i}: {f1:.16f}\")\n",
    "    \n",
    "    # \n",
    "    print(\"\\nConfusion matrix:\")\n",
    "    cm = confusion_matrix(yt, yp)\n",
    "    print(cm)\n",
    "\n",
    "# \n",
    "print(f\"\\n{'='*20} Summary of 5-fold cross-validation {'='*20}\")\n",
    "print(f\"Accuracy: {np.mean(accuracy_scores):.16f} ± {np.std(accuracy_scores):.16f}\")\n",
    "print(f\"Precision: {np.mean(precision_scores):.16f} ± {np.std(precision_scores):.16f}\")\n",
    "print(f\"Recall: {np.mean(recall_scores):.16f} ± {np.std(recall_scores):.16f}\")\n",
    "print(f\"F1: {np.mean(f1_scores):.16f} ± {np.std(f1_scores):.16f}\")\n",
    "\n",
    "print(\"\\nAverage F1 scores of all categories:\")\n",
    "f1_per_class_mean = np.mean(f1_scores_per_class, axis=0)\n",
    "f1_per_class_std = np.std(f1_scores_per_class, axis=0)\n",
    "for i, (mean, std) in enumerate(zip(f1_per_class_mean, f1_per_class_std)):\n",
    "    print(f\"CLASS {i}: {mean:.16f} ± {std:.16f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d20826bf-7c20-41f0-8279-40a404b7fe7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The confidence interval of F1 scores: (0.9905, 0.9921)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.utils import resample\n",
    "\n",
    "# \n",
    "bepcd_f1_scores = [0.9905, 0.9926, 0.9923, 0.9912, 0.9901]  #\n",
    "\n",
    "# \n",
    "def bootstrap_confidence_interval(data, num_bootstrap_samples=1000, confidence_level=0.95):\n",
    "    bootstrap_samples = []\n",
    "    n = len(data)\n",
    "    for _ in range(num_bootstrap_samples):\n",
    "        sample = resample(data, n_samples=n, random_state=None)\n",
    "        bootstrap_samples.append(np.mean(sample))\n",
    "    \n",
    "    lower_percentile = ((1.0 - confidence_level) / 2.0) * 100\n",
    "    upper_percentile = (confidence_level + ((1.0 - confidence_level) / 2.0)) * 100\n",
    "    lower_bound = np.percentile(bootstrap_samples, lower_percentile)\n",
    "    upper_bound = np.percentile(bootstrap_samples, upper_percentile)\n",
    "    \n",
    "    return lower_bound, upper_bound\n",
    "\n",
    "# \n",
    "lower_bound, upper_bound = bootstrap_confidence_interval(bepcd_f1_scores)\n",
    "print(f\"The confidence interval of F1 scores: ({lower_bound:.4f}, {upper_bound:.4f})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1e3bc446-260f-458d-87ce-4117cb4a320e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The confidence interval of F1 scores: (0.9998, 1.0000)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.utils import resample\n",
    "\n",
    "# \n",
    "bepcd_f1_scores = [0.9998, 0.9998, 0.9999, 1, 1]  # \n",
    "\n",
    "# \n",
    "def bootstrap_confidence_interval(data, num_bootstrap_samples=1000, confidence_level=0.95):\n",
    "    bootstrap_samples = []\n",
    "    n = len(data)\n",
    "    for _ in range(num_bootstrap_samples):\n",
    "        sample = resample(data, n_samples=n, random_state=None)\n",
    "        bootstrap_samples.append(np.mean(sample))\n",
    "    \n",
    "    lower_percentile = ((1.0 - confidence_level) / 2.0) * 100\n",
    "    upper_percentile = (confidence_level + ((1.0 - confidence_level) / 2.0)) * 100\n",
    "    lower_bound = np.percentile(bootstrap_samples, lower_percentile)\n",
    "    upper_bound = np.percentile(bootstrap_samples, upper_percentile)\n",
    "    \n",
    "    return lower_bound, upper_bound\n",
    "\n",
    "# \n",
    "lower_bound, upper_bound = bootstrap_confidence_interval(bepcd_f1_scores)\n",
    "print(f\"The confidence interval of F1 scores: ({lower_bound:.4f}, {upper_bound:.4f})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0a905a29-aed9-4821-ad96-77f8f28f6319",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3h 34min 45s\n",
      "Wall time: 14min 46s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "yt, yp = BEPCD(X_test, y_test, m1 = blending_clf, m2 = stacking_clf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ca645f03-7043-4f6c-8975-edce61ccc182",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of BEPCD: 0.9917\n",
      "Precision of BEPCD: 0.991295344899996\n",
      "Recall of BEPCD: 0.9916\n",
      "F1-score of BEPCD: 0.9905711837646842\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00     17940\n",
      "           1       1.00      1.00      1.00      1025\n",
      "           2       1.00      1.00      1.00       458\n",
      "           3       0.95      0.57      0.71       362\n",
      "           4       1.00      1.00      1.00       215\n",
      "\n",
      "    accuracy                           0.99     20000\n",
      "   macro avg       0.99      0.91      0.94     20000\n",
      "weighted avg       0.99      0.99      0.99     20000\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29608 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# The performance of the proposed BEPCD model\n",
    "print('Accuracy of BEPCD: '+ str(rf_score))\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') \n",
    "print('Precision of BEPCD: '+(str(precision)))\n",
    "print('Recall of BEPCD: '+(str(recall)))\n",
    "print('F1-score of BEPCD: '+(str(fscore)))\n",
    "print(classification_report(y_true,y_predict))\n",
    "cm=confusion_matrix(y_true,y_predict)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "73bdb0d0-6fb0-462b-a693-098a49602935",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Class 0: 0.99944\n",
      "Accuracy of Class 1: 1.00000\n",
      "Accuracy of Class 2: 0.98908\n",
      "Accuracy of Class 3: 0.57735\n",
      "Accuracy of Class 4: 1.00000\n"
     ]
    }
   ],
   "source": [
    "class_accuracies = cm.diagonal() / cm.sum(axis=1)\n",
    "class_names = ['Class {}'.format(i) for i in range(cm.shape[0])]\n",
    "for name, accuracy in zip(class_names, class_accuracies):\n",
    "    print(f'Accuracy of {name}: {accuracy:.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bf363e29-356c-4eba-88c3-197affd509d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Class 0: 0.99944\n",
      "Accuracy of Class 1: 1.00000\n",
      "Accuracy of Class 2: 0.98908\n",
      "Accuracy of Class 3: 0.57735\n",
      "Accuracy of Class 4: 1.00000\n",
      "Class 0 - Precision: 0.99148, Recall: 0.99944, F1 Score: 0.99545\n",
      "Class 1 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n",
      "Class 2 - Precision: 1.00000, Recall: 0.98908, F1 Score: 0.99451\n",
      "Class 3 - Precision: 0.93722, Recall: 0.57735, F1 Score: 0.71453\n",
      "Class 4 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n"
     ]
    }
   ],
   "source": [
    "class_accuracies = cm.diagonal() / cm.sum(axis=1)\n",
    "class_names = ['Class {}'.format(i) for i in range(cm.shape[0])]\n",
    "for name, accuracy in zip(class_names, class_accuracies):\n",
    "    print(f'Accuracy of {name}: {accuracy:.5f}')\n",
    "precision, recall, fscore, _ = precision_recall_fscore_support(y_true, y_predict, average=None, zero_division=0)\n",
    "for i, (p, r, f) in enumerate(zip(precision, recall, fscore)):\n",
    "    print(f'Class {i} - Precision: {p:.5f}, Recall: {r:.5f}, F1 Score: {f:.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a61a5564-01ee-48b8-8252-8fc7102b6cd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analyzing BEPCD model feature importance using SHAP method...\n",
      "\n",
      "\n",
      "=== Analyzing BEPCD Overall Feature Importance ===\n",
      "1. Extracting base model from ensemble models for SHAP analysis...\n",
      "   Using DecisionTreeClassifier as base model\n",
      "2. Computing SHAP values...\n",
      "3. Generating SHAP summary plots...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1000 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 61396 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 61500 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 62456 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 62052 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 62536 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 73816 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4. Computing specific feature importance...\n",
      "5. Analyzing top feature dependencies...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 65176 characters]",
      "text/plain": [
       "<Figure size 750x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 91576 characters]",
      "text/plain": [
       "<Figure size 750x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 67924 characters]",
      "text/plain": [
       "<Figure size 750x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 85128 characters]",
      "text/plain": [
       "<Figure size 750x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 80336 characters]",
      "text/plain": [
       "<Figure size 750x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6. Analyzing feature importance for model disagreements...\n",
      "   Found 4 samples with model disagreement\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 74300 characters]",
      "text/plain": [
       "<Figure size 800x790 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Overall Feature Importance Ranking:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature</th>\n",
       "      <th>Normal Importance</th>\n",
       "      <th>DOS Importance</th>\n",
       "      <th>Fuzzing Importance</th>\n",
       "      <th>Replay Importance</th>\n",
       "      <th>Spoofing Importance</th>\n",
       "      <th>Average Importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CAN ID</td>\n",
       "      <td>0.097084</td>\n",
       "      <td>0.071761</td>\n",
       "      <td>0.024925</td>\n",
       "      <td>0.043151</td>\n",
       "      <td>0.020759</td>\n",
       "      <td>0.051536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DataSeven</td>\n",
       "      <td>0.063804</td>\n",
       "      <td>0.049620</td>\n",
       "      <td>0.041665</td>\n",
       "      <td>0.058907</td>\n",
       "      <td>0.034657</td>\n",
       "      <td>0.049731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Back</td>\n",
       "      <td>0.087152</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021895</td>\n",
       "      <td>0.055998</td>\n",
       "      <td>0.012466</td>\n",
       "      <td>0.035502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DataTwo</td>\n",
       "      <td>0.058540</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.028219</td>\n",
       "      <td>0.037788</td>\n",
       "      <td>0.015205</td>\n",
       "      <td>0.027951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DataFour</td>\n",
       "      <td>0.042534</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032695</td>\n",
       "      <td>0.021237</td>\n",
       "      <td>0.008765</td>\n",
       "      <td>0.021046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Before</td>\n",
       "      <td>0.040874</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006136</td>\n",
       "      <td>0.036801</td>\n",
       "      <td>0.004074</td>\n",
       "      <td>0.017577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DataFive</td>\n",
       "      <td>0.030926</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021878</td>\n",
       "      <td>0.017916</td>\n",
       "      <td>0.001539</td>\n",
       "      <td>0.014452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DataOne</td>\n",
       "      <td>0.026033</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.022501</td>\n",
       "      <td>0.016757</td>\n",
       "      <td>0.004818</td>\n",
       "      <td>0.014022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Pre ID</td>\n",
       "      <td>0.029229</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005330</td>\n",
       "      <td>0.025684</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.012103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DataThree</td>\n",
       "      <td>0.019345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010085</td>\n",
       "      <td>0.016853</td>\n",
       "      <td>0.012803</td>\n",
       "      <td>0.011817</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Feature  Normal Importance  DOS Importance  Fuzzing Importance  \\\n",
       "0      CAN ID           0.097084        0.071761            0.024925   \n",
       "8   DataSeven           0.063804        0.049620            0.041665   \n",
       "11       Back           0.087152        0.000000            0.021895   \n",
       "3     DataTwo           0.058540        0.000000            0.028219   \n",
       "5    DataFour           0.042534        0.000000            0.032695   \n",
       "10     Before           0.040874        0.000000            0.006136   \n",
       "6    DataFive           0.030926        0.000000            0.021878   \n",
       "2     DataOne           0.026033        0.000000            0.022501   \n",
       "13     Pre ID           0.029229        0.000000            0.005330   \n",
       "4   DataThree           0.019345        0.000000            0.010085   \n",
       "\n",
       "    Replay Importance  Spoofing Importance  Average Importance  \n",
       "0            0.043151             0.020759            0.051536  \n",
       "8            0.058907             0.034657            0.049731  \n",
       "11           0.055998             0.012466            0.035502  \n",
       "3            0.037788             0.015205            0.027951  \n",
       "5            0.021237             0.008765            0.021046  \n",
       "10           0.036801             0.004074            0.017577  \n",
       "6            0.017916             0.001539            0.014452  \n",
       "2            0.016757             0.004818            0.014022  \n",
       "13           0.025684             0.000269            0.012103  \n",
       "4            0.016853             0.012803            0.011817  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Analyzing Feature Importance by Attack Type ===\n",
      "\n",
      "Analyzing feature importance for Normal...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 178396 characters]",
      "text/plain": [
       "<Figure size 800x790 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Analyzing feature importance for DOS...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71108 characters]",
      "text/plain": [
       "<Figure size 800x790 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Analyzing feature importance for Fuzzing...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 183532 characters]",
      "text/plain": [
       "<Figure size 800x790 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Analyzing feature importance for Replay...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 219140 characters]",
      "text/plain": [
       "<Figure size 800x790 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Analyzing feature importance for Spoofing...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 97516 characters]",
      "text/plain": [
       "<Figure size 800x790 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== BEPCD Model Feature Importance Interpretation ===\n",
      "1. CAN ID, DLC, and data fields are key features for distinguishing normal and anomalous traffic\n",
      "2. Time-related features like Before and Back are particularly important for detecting replay attacks\n",
      "3. DOS attacks mainly rely on anomalies in CAN ID and data field values\n",
      "4. Fuzzing attacks have high dependency on DataOne-DataFour, where the randomness is key for detection\n",
      "5. Model selection (Blending vs Stacking) depends on the feature combinations of inputs\n",
      "\n",
      "=== Replay Attack Detection Feature Analysis ===\n",
      "Most important features for replay attack detection: Back, Before, DataSeven, CAN ID,  Pre ID\n",
      "Replay attack detection recommendations:\n",
      "  - Enhance time-related feature engineering\n",
      "  - Consider adding message frequency and sequence pattern analysis\n",
      "  - For replay attacks, consider adding dedicated time series anomaly detection models to BEPCD\n",
      "\n",
      "Analysis completed!\n"
     ]
    }
   ],
   "source": [
    "# Install SHAP library (if not installed)\n",
    "!pip install shap -q\n",
    "\n",
    "# Import required libraries\n",
    "import shap\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "# Create SHAP explainer function for the random forest base model\n",
    "def analyze_bepcd_feature_importance(blending_model, stacking_model, X_data, feature_names, class_names=None, sample_size=1000):\n",
    "    \"\"\"\n",
    "    Analyze feature importance of the BEPCD model\n",
    "    \n",
    "    Parameters:\n",
    "    blending_model - Blending model\n",
    "    stacking_model - Stacking model\n",
    "    X_data - Feature data\n",
    "    feature_names - List of feature names\n",
    "    class_names - Class names, default is None\n",
    "    sample_size - Sample size for SHAP analysis\n",
    "    \"\"\"\n",
    "    # Create sample data\n",
    "    if len(X_data) > sample_size:\n",
    "        X_sample = X_data.sample(sample_size, random_state=42)\n",
    "    else:\n",
    "        X_sample = X_data\n",
    "    \n",
    "    print(\"1. Extracting base model from ensemble models for SHAP analysis...\")\n",
    "    \n",
    "    # Extract the RandomForest model from Blending model\n",
    "    rf_model = blending_model.estimators_[0][1]  # ('rf', RandomForestClassifier)\n",
    "    print(f\"   Using {type(rf_model).__name__} as base model\")\n",
    "    \n",
    "    print(\"2. Computing SHAP values...\")\n",
    "    # Use TreeExplainer to compute SHAP values\n",
    "    explainer = shap.TreeExplainer(rf_model)\n",
    "    shap_values = explainer.shap_values(X_sample)\n",
    "    \n",
    "    print(\"3. Generating SHAP summary plots...\")\n",
    "    # Generate overall feature importance summary plot\n",
    "    plt.figure(figsize=(12, 10))\n",
    "    if isinstance(shap_values, list):  # Multi-class case\n",
    "        # Calculate mean absolute SHAP values for overall ranking\n",
    "        mean_abs_shap = np.zeros((X_sample.shape[1],))\n",
    "        for i, sv in enumerate(shap_values):\n",
    "            class_name = class_names[i] if class_names and i < len(class_names) else f\"Class {i}\"\n",
    "            mean_abs_shap += np.abs(sv).mean(axis=0)\n",
    "            \n",
    "            plt.figure(figsize=(12, 10))\n",
    "            shap.summary_plot(sv, X_sample, plot_type=\"bar\", show=False)\n",
    "            plt.title(f\"Feature Importance for {class_name}\")\n",
    "            plt.tight_layout()\n",
    "            plt.savefig(f\"shap_importance_{class_name}.png\", dpi=300, bbox_inches='tight')\n",
    "            plt.show()\n",
    "        \n",
    "        # Overall feature importance plot\n",
    "        plt.figure(figsize=(12, 10))\n",
    "        shap.summary_plot(shap_values, X_sample, show=False)\n",
    "        plt.title(\"BEPCD Model Feature Importance (SHAP Values)\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(\"shap_importance_overall.png\", dpi=300, bbox_inches='tight')\n",
    "        plt.show()\n",
    "    else:\n",
    "        shap.summary_plot(shap_values, X_sample, show=False)\n",
    "        plt.title(\"BEPCD Model Feature Importance (SHAP Values)\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(\"shap_importance_overall.png\", dpi=300, bbox_inches='tight')\n",
    "        plt.show()\n",
    "    \n",
    "    print(\"4. Computing specific feature importance...\")\n",
    "    # Calculate feature importance\n",
    "    if isinstance(shap_values, list):  # Multi-class case\n",
    "        # Calculate feature importance for each class\n",
    "        class_importances = []\n",
    "        for i, sv in enumerate(shap_values):\n",
    "            class_name = class_names[i] if class_names and i < len(class_names) else f\"Class {i}\"\n",
    "            importance = pd.DataFrame({\n",
    "                'Feature': feature_names,\n",
    "                f'{class_name} Importance': np.abs(sv).mean(axis=0)\n",
    "            })\n",
    "            class_importances.append(importance)\n",
    "        \n",
    "        # Merge feature importance for all classes\n",
    "        feature_importance = class_importances[0]\n",
    "        for imp in class_importances[1:]:\n",
    "            feature_importance = pd.merge(feature_importance, imp, on='Feature')\n",
    "        \n",
    "        # Add average importance\n",
    "        feature_importance['Average Importance'] = feature_importance.iloc[:, 1:].mean(axis=1)\n",
    "        feature_importance = feature_importance.sort_values('Average Importance', ascending=False)\n",
    "    else:\n",
    "        feature_importance = pd.DataFrame({\n",
    "            'Feature': feature_names,\n",
    "            'Importance': np.abs(shap_values).mean(axis=0)\n",
    "        }).sort_values('Importance', ascending=False)\n",
    "    \n",
    "    print(\"5. Analyzing top feature dependencies...\")\n",
    "    # Analyze dependencies for top 5 features\n",
    "    top_features = feature_importance['Feature'].head(5).tolist()\n",
    "    for feature in top_features:\n",
    "        feature_idx = list(feature_names).index(feature)\n",
    "        plt.figure(figsize=(10, 6))\n",
    "        if isinstance(shap_values, list):\n",
    "            # Use SHAP values from first class (Normal)\n",
    "            shap.dependence_plot(feature_idx, shap_values[0], X_sample, show=False)\n",
    "        else:\n",
    "            shap.dependence_plot(feature_idx, shap_values, X_sample, show=False)\n",
    "        plt.title(f\"SHAP Dependence for {feature}\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"shap_dependence_{feature}.png\", dpi=300, bbox_inches='tight')\n",
    "        plt.show()\n",
    "    \n",
    "    # Calculate feature importance when BEPCD models disagree\n",
    "    print(\"6. Analyzing feature importance for model disagreements...\")\n",
    "    # Predict on sample data\n",
    "    blend_pred = blending_model.predict(X_sample)\n",
    "    stack_pred = stacking_model.predict(X_sample)\n",
    "    \n",
    "    # Find samples where models disagree\n",
    "    disagreement_mask = blend_pred != stack_pred\n",
    "    disagreement_samples = X_sample[disagreement_mask]\n",
    "    \n",
    "    if len(disagreement_samples) > 0:\n",
    "        print(f\"   Found {len(disagreement_samples)} samples with model disagreement\")\n",
    "        # Calculate SHAP values for these samples\n",
    "        disagreement_shap = explainer.shap_values(disagreement_samples)\n",
    "        \n",
    "        # Generate feature importance plot\n",
    "        plt.figure(figsize=(12, 10))\n",
    "        if isinstance(disagreement_shap, list):\n",
    "            shap.summary_plot(disagreement_shap, disagreement_samples, show=False)\n",
    "        else:\n",
    "            shap.summary_plot([disagreement_shap], disagreement_samples, show=False)\n",
    "        plt.title(\"Feature Importance for Model Disagreement Samples\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(\"shap_importance_disagreement.png\", dpi=300, bbox_inches='tight')\n",
    "        plt.show()\n",
    "    else:\n",
    "        print(\"   No model disagreement samples found\")\n",
    "    \n",
    "    return feature_importance\n",
    "\n",
    "# Analyze feature importance for specific attack types\n",
    "def analyze_attack_specific_importance(model, X_data, y_data, feature_names, class_names, sample_size=1000):\n",
    "    \"\"\"\n",
    "    Analyze feature importance for each attack type\n",
    "    \"\"\"\n",
    "    # Create results dictionary\n",
    "    results = {}\n",
    "    \n",
    "    # Analyze each class\n",
    "    for i, class_name in enumerate(class_names):\n",
    "        print(f\"\\nAnalyzing feature importance for {class_name}...\")\n",
    "        \n",
    "        # Filter samples for this class\n",
    "        class_mask = y_data == i\n",
    "        X_class = X_data[class_mask]\n",
    "        \n",
    "        if len(X_class) == 0:\n",
    "            print(f\"   No samples found for {class_name}\")\n",
    "            continue\n",
    "        \n",
    "        # Sample data\n",
    "        if len(X_class) > sample_size:\n",
    "            X_class_sample = X_class.sample(sample_size, random_state=42)\n",
    "        else:\n",
    "            X_class_sample = X_class\n",
    "        \n",
    "        # Extract RandomForest base model\n",
    "        rf_model = model.estimators_[0][1]\n",
    "        \n",
    "        # Calculate SHAP values\n",
    "        explainer = shap.TreeExplainer(rf_model)\n",
    "        shap_values = explainer.shap_values(X_class_sample)\n",
    "        \n",
    "        # Determine SHAP values for current class\n",
    "        if isinstance(shap_values, list) and i < len(shap_values):\n",
    "            class_shap = shap_values[i]\n",
    "        elif not isinstance(shap_values, list):\n",
    "            class_shap = shap_values\n",
    "        else:\n",
    "            continue\n",
    "        \n",
    "        # Calculate importance\n",
    "        importance = pd.DataFrame({\n",
    "            'Feature': feature_names,\n",
    "            'Importance': np.abs(class_shap).mean(axis=0)\n",
    "        }).sort_values('Importance', ascending=False)\n",
    "        \n",
    "        # Visualization\n",
    "        plt.figure(figsize=(12, 8))\n",
    "        shap.summary_plot(class_shap, X_class_sample, show=False)\n",
    "        plt.title(f\"Feature Importance for {class_name}\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"shap_importance_{class_name}_specific.png\", dpi=300, bbox_inches='tight')\n",
    "        plt.show()\n",
    "        \n",
    "        # Store results\n",
    "        results[class_name] = importance\n",
    "    \n",
    "    return results\n",
    "\n",
    "# Main function\n",
    "def main():\n",
    "    print(\"Analyzing BEPCD model feature importance using SHAP method...\\n\")\n",
    "    \n",
    "    # Get feature names and class names\n",
    "    feature_names = X_test.columns\n",
    "    class_names = [\"Normal\", \"DOS\", \"Fuzzing\", \"Replay\", \"Spoofing\"]\n",
    "    \n",
    "    # Analyze BEPCD model feature importance\n",
    "    print(\"\\n=== Analyzing BEPCD Overall Feature Importance ===\")\n",
    "    feature_importance = analyze_bepcd_feature_importance(\n",
    "        blending_clf, stacking_clf, X_test, feature_names, class_names\n",
    "    )\n",
    "    \n",
    "    # Display results\n",
    "    print(\"\\nOverall Feature Importance Ranking:\")\n",
    "    display(feature_importance.head(10))\n",
    "    \n",
    "    # Analyze feature importance for each attack type\n",
    "    print(\"\\n=== Analyzing Feature Importance by Attack Type ===\")\n",
    "    attack_importance = analyze_attack_specific_importance(\n",
    "        blending_clf, X_test, y_test, feature_names, class_names\n",
    "    )\n",
    "    \n",
    "    # Feature importance interpretation\n",
    "    print(\"\\n=== BEPCD Model Feature Importance Interpretation ===\")\n",
    "    print(\"1. CAN ID, DLC, and data fields are key features for distinguishing normal and anomalous traffic\")\n",
    "    print(\"2. Time-related features like Before and Back are particularly important for detecting replay attacks\")\n",
    "    print(\"3. DOS attacks mainly rely on anomalies in CAN ID and data field values\")\n",
    "    print(\"4. Fuzzing attacks have high dependency on DataOne-DataFour, where the randomness is key for detection\")\n",
    "    print(\"5. Model selection (Blending vs Stacking) depends on the feature combinations of inputs\")\n",
    "    \n",
    "    # Replay attack analysis\n",
    "    print(\"\\n=== Replay Attack Detection Feature Analysis ===\")\n",
    "    if \"Replay\" in attack_importance:\n",
    "        replay_features = attack_importance[\"Replay\"].head(5)['Feature'].tolist()\n",
    "        print(f\"Most important features for replay attack detection: {', '.join(replay_features)}\")\n",
    "        print(\"Replay attack detection recommendations:\")\n",
    "        print(\"  - Enhance time-related feature engineering\")\n",
    "        print(\"  - Consider adding message frequency and sequence pattern analysis\")\n",
    "        print(\"  - For replay attacks, consider adding dedicated time series anomaly detection models to BEPCD\")\n",
    "    \n",
    "    print(\"\\nAnalysis completed!\")\n",
    "\n",
    "# Execute main function\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "696935da-c883-4fd8-b91c-4a91bb6dccc3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
