{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a11bab8a-922b-4ce7-b8b8-3c40a45d2a97",
   "metadata": {},
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c7b08ce1-1cab-400b-8035-e7266fe04586",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6b9f0d06-4a2a-4eea-a2de-275643bd9072",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score,precision_recall_fscore_support\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "import xgboost as xgb\n",
    "import time\n",
    "from river import stream\n",
    "from statistics import mode\n",
    "from statistics import mode\n",
    "from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from statistics import mode\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LassoCV\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, roc_curve, auc, log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import joblib\n",
    "from joblib import load\n",
    "import os\n",
    "from sklearn.preprocessing import PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a500ef73-6ed4-4e58-8d03-a67f0800e45b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/CICIOV_2024.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a7566206-5ccf-41d8-b621-25dc04643086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DATA_0</th>\n",
       "      <th>DATA_1</th>\n",
       "      <th>DATA_2</th>\n",
       "      <th>DATA_3</th>\n",
       "      <th>DATA_4</th>\n",
       "      <th>DATA_5</th>\n",
       "      <th>DATA_6</th>\n",
       "      <th>DATA_7</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>936</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>359</td>\n",
       "      <td>0</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>227</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>344</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>57</td>\n",
       "      <td>6</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>516</td>\n",
       "      <td>192</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>531</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>8</td>\n",
       "      <td>127</td>\n",
       "      <td>47</td>\n",
       "      <td>7</td>\n",
       "      <td>255</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99983</th>\n",
       "      <td>532</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99984</th>\n",
       "      <td>513</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>156</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99985</th>\n",
       "      <td>514</td>\n",
       "      <td>28</td>\n",
       "      <td>239</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99986</th>\n",
       "      <td>128</td>\n",
       "      <td>132</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>138</td>\n",
       "      <td>34</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99987</th>\n",
       "      <td>531</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>8</td>\n",
       "      <td>127</td>\n",
       "      <td>47</td>\n",
       "      <td>7</td>\n",
       "      <td>255</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99988 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID  DATA_0  DATA_1  DATA_2  DATA_3  DATA_4  DATA_5  DATA_6  DATA_7  \\\n",
       "0      936       1       0      39      16       0       0       0       0   \n",
       "1      359       0     128       0       0       0       1     227       0   \n",
       "2      344       6      20       6      57       6      33       2      17   \n",
       "3      516     192       0     125       0       0       0       0       0   \n",
       "4      531     255     255       8     127      47       7     255      80   \n",
       "...    ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "99983  532       0       0       0       0       0       0     255       0   \n",
       "99984  513       0       0       0       0       0       0      64     156   \n",
       "99985  514      28     239     112       0      64       0       0       0   \n",
       "99986  128     132       3       2      35      24       5     138      34   \n",
       "99987  531     255     255       8     127      47       7     255      80   \n",
       "\n",
       "       Label  \n",
       "0          0  \n",
       "1          0  \n",
       "2          4  \n",
       "3          0  \n",
       "4          0  \n",
       "...      ...  \n",
       "99983      0  \n",
       "99984      2  \n",
       "99985      0  \n",
       "99986      5  \n",
       "99987      0  \n",
       "\n",
       "[99988 rows x 10 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e46a8255-c352-453e-9417-9c76edf37fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    50000\n",
       "1    10000\n",
       "3    10000\n",
       "4     9999\n",
       "5     9998\n",
       "2     9991\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "dae95f06-e65e-4e27-a9c5-93967c08169a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(['Label'],axis=1)\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, train_size = 0.8, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "24786bdd-8b24-46ca-8c2b-76b77efd70ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    40026\n",
       "5     8023\n",
       "3     8019\n",
       "2     7980\n",
       "1     7975\n",
       "4     7967\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "783d53b4-5c72-430f-ac36-1e79bd16954b",
   "metadata": {},
   "source": [
    "0 BENIGN \\\n",
    "1 DOS\\\n",
    "2 Spoofing_GAS\\\n",
    "3 Spoofing_RPM\\\n",
    "4 Spoofing_Speed\\\n",
    "5 Spoofing_STEERING_WHEEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f10573c7-c986-43b9-b1f4-db3e77bb2ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote=SMOTE(n_jobs=-1,sampling_strategy={1:10000,2:10000,3:10000,4:10000,5:10000})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d209155a-1dfb-4d61-8c44-8ff2515e7868",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "693d9ba1-b782-4502-b59d-3ae25ec1faa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label\n",
       "0    40026\n",
       "3    10000\n",
       "1    10000\n",
       "2    10000\n",
       "5    10000\n",
       "4    10000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y_train).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b41d2827-bccd-441a-aaa1-74c4ca49ad51",
   "metadata": {},
   "source": [
    "# Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6e93ce5-69a9-451e-b34d-4344cf401ac6",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "74f28ef8-d571-47e8-96c9-62cb7dafadf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 2.12 s\n",
      "Wall time: 2.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "rf = RandomForestClassifier(random_state = 0)\n",
    "rf.fit(X_train,y_train.astype('int')) \n",
    "\n",
    "rf_score=rf.score(X_test,y_test.astype('int'))\n",
    "\n",
    "y_predict=rf.predict(X_test)\n",
    "y_true=y_test.astype('int')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6f42fb8-a981-4217-ace1-c31385128e22",
   "metadata": {},
   "source": [
    "## ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6b7a5eb9-7f2e-40fd-8246-8a3a7341ad5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1.8 s\n",
      "Wall time: 1.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Extra trees training and prediction\n",
    "et = ExtraTreesClassifier(random_state = 0)\n",
    "\n",
    "et.fit(X_train,y_train.astype('int')) \n",
    "\n",
    "et_score=et.score(X_test,y_test.astype('int'))\n",
    "\n",
    "y_predict=et.predict(X_test)\n",
    "\n",
    "y_true=y_test.astype('int')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "558a930b-c7c7-434e-a6a9-641730af85e7",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "7f403fbb-4212-404f-9467-c5795059d746",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 8.3 s\n",
      "Wall time: 561 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the XGBoost algorithm\n",
    "import xgboost as xgb\n",
    "xg = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss')\n",
    "\n",
    "X_train_x = X_train.values\n",
    "X_test_x = X_test.values\n",
    "\n",
    "xg.fit(X_train_x, y_train)\n",
    "\n",
    "y_pred = xg.predict(X_test_x)\n",
    "\n",
    "xg_f1=f1_score(y_test, y_pred, average=None)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e68fcd5-69dc-459e-b647-06bc1683aec5",
   "metadata": {},
   "source": [
    "## CATBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "66ae4693-c399-4a77-a3ca-5398d1bfecdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3min 6s\n",
      "Wall time: 17.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the CatBoost algorithm\n",
    "import catboost as cbt\n",
    "cb = cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')\n",
    "\n",
    "#cb = cbt.CatBoostClassifier()\n",
    "\n",
    "cb.fit(X_train, y_train)\n",
    "# 创建并训练CatBoost分类器\n",
    "y_pred = cb.predict(X_test)\n",
    "\n",
    "\n",
    "cb_f1=f1_score(y_test, y_pred, average=None)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a43a19b-787e-458c-8b9e-56b5ed8f216e",
   "metadata": {},
   "source": [
    "## Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "439d9134-9c47-4924-bc19-4ae8b03941cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1min 33s\n",
      "Wall time: 24.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "base_classifiers = [\n",
    "    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "    ('et', ExtraTreesClassifier(n_estimators=100, random_state=42)),\n",
    "    ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "]\n",
    "\n",
    "\n",
    "meta_classifier = cbt.CatBoostClassifier(n_estimators=100, random_state=42, verbose=0)\n",
    "\n",
    "\n",
    "stacking_clf = StackingClassifier(estimators=base_classifiers, final_estimator=meta_classifier)\n",
    "\n",
    "stacking_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = stacking_clf.predict(X_test)\n",
    "\n",
    "stacking_f1 = f1_score(y_test, y_pred, average=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4df4d6d-13bc-48c7-9ee9-2ff363f3912e",
   "metadata": {},
   "source": [
    "## BLending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6c35de35-8a2d-49a4-876b-13fc73d23895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3min 13s\n",
      "Wall time: 19.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "\n",
    "base_classifiers = [\n",
    "    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "    ('cb', cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')),\n",
    "    ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "]\n",
    "\n",
    "\n",
    "blending_clf = VotingClassifier(estimators=base_classifiers, weights=None, voting='soft')\n",
    "\n",
    "blending_clf.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = blending_clf.predict(X_test)\n",
    "\n",
    "\n",
    "blending_f1 = f1_score(y_test, y_pred, average=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e23f7a8d-063f-4e09-ad79-7829cb29221f",
   "metadata": {},
   "source": [
    "# BEPCD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "292fb011-b9a1-44f3-9517-b958556368aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=[]\n",
    "for i in range(len(blending_f1)):\n",
    "    if max(blending_f1[i],stacking_f1[i]) == stacking_f1[i]:\n",
    "        model.append(stacking_clf)\n",
    "    else:\n",
    "        model.append(blending_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "7d1e846d-99a3-42df-a085-e885cf8a5b38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "==================== 第 1 折验证 ====================\n",
      "训练集大小: 90092, 验证集大小: 19998\n",
      "\n",
      "当前折的评估结果:\n",
      "准确率: 1.0000000000000000\n",
      "精确率: 1.0000000000000000\n",
      "召回率: 1.0000000000000000\n",
      "F1分数: 1.0000000000000000\n",
      "\n",
      "各类别的F1分数:\n",
      "类别 0: 1.0000000000000000\n",
      "类别 1: 1.0000000000000000\n",
      "类别 2: 1.0000000000000000\n",
      "类别 3: 1.0000000000000000\n",
      "类别 4: 1.0000000000000000\n",
      "类别 5: 1.0000000000000000\n",
      "\n",
      "混淆矩阵:\n",
      "[[9908    0    0    0    0    0]\n",
      " [   0 2028    0    0    0    0]\n",
      " [   0    0 1992    0    0    0]\n",
      " [   0    0    0 2067    0    0]\n",
      " [   0    0    0    0 2019    0]\n",
      " [   0    0    0    0    0 1984]]\n",
      "\n",
      "==================== 第 2 折验证 ====================\n",
      "训练集大小: 90098, 验证集大小: 19998\n",
      "\n",
      "当前折的评估结果:\n",
      "准确率: 0.9998999899989999\n",
      "精确率: 0.9999000199265897\n",
      "召回率: 0.9998999899989999\n",
      "F1分数: 0.9998999733021409\n",
      "\n",
      "各类别的F1分数:\n",
      "类别 0: 0.9999495077000757\n",
      "类别 1: 0.9997511818860414\n",
      "类别 2: 1.0000000000000000\n",
      "类别 3: 0.9995164410058027\n",
      "类别 4: 1.0000000000000000\n",
      "类别 5: 1.0000000000000000\n",
      "\n",
      "混淆矩阵:\n",
      "[[9902    0    0    0    0    0]\n",
      " [   0 2009    0    0    0    0]\n",
      " [   0    0 2004    0    0    0]\n",
      " [   1    1    0 2067    0    0]\n",
      " [   0    0    0    0 2010    0]\n",
      " [   0    0    0    0    0 2004]]\n",
      "\n",
      "==================== 第 3 折验证 ====================\n",
      "训练集大小: 89888, 验证集大小: 19998\n",
      "\n",
      "当前折的评估结果:\n",
      "准确率: 0.9999499949995000\n",
      "精确率: 0.9999499999441258\n",
      "召回率: 0.9999499949995000\n",
      "F1分数: 0.9999499898198848\n",
      "\n",
      "各类别的F1分数:\n",
      "类别 0: 0.9999505562422745\n",
      "类别 1: 0.9997433923530921\n",
      "类别 2: 1.0000000000000000\n",
      "类别 3: 1.0000000000000000\n",
      "类别 4: 1.0000000000000000\n",
      "类别 5: 1.0000000000000000\n",
      "\n",
      "混淆矩阵:\n",
      "[[10112     0     0     0     0     0]\n",
      " [    1  1948     0     0     0     0]\n",
      " [    0     0  1972     0     0     0]\n",
      " [    0     0     0  1993     0     0]\n",
      " [    0     0     0     0  1999     0]\n",
      " [    0     0     0     0     0  1973]]\n",
      "\n",
      "==================== 第 4 折验证 ====================\n",
      "训练集大小: 89862, 验证集大小: 19997\n",
      "\n",
      "当前折的评估结果:\n",
      "准确率: 1.0000000000000000\n",
      "精确率: 1.0000000000000000\n",
      "召回率: 1.0000000000000000\n",
      "F1分数: 1.0000000000000000\n",
      "\n",
      "各类别的F1分数:\n",
      "类别 0: 1.0000000000000000\n",
      "类别 1: 1.0000000000000000\n",
      "类别 2: 1.0000000000000000\n",
      "类别 3: 1.0000000000000000\n",
      "类别 4: 1.0000000000000000\n",
      "类别 5: 1.0000000000000000\n",
      "\n",
      "混淆矩阵:\n",
      "[[10138     0     0     0     0     0]\n",
      " [    0  1970     0     0     0     0]\n",
      " [    0     0  1967     0     0     0]\n",
      " [    0     0     0  1944     0     0]\n",
      " [    0     0     0     0  1984     0]\n",
      " [    0     0     0     0     0  1994]]\n",
      "\n",
      "==================== 第 5 折验证 ====================\n",
      "训练集大小: 90060, 验证集大小: 19997\n",
      "\n",
      "当前折的评估结果:\n",
      "准确率: 1.0000000000000000\n",
      "精确率: 1.0000000000000000\n",
      "召回率: 1.0000000000000000\n",
      "F1分数: 1.0000000000000000\n",
      "\n",
      "各类别的F1分数:\n",
      "类别 0: 1.0000000000000000\n",
      "类别 1: 1.0000000000000000\n",
      "类别 2: 1.0000000000000000\n",
      "类别 3: 1.0000000000000000\n",
      "类别 4: 1.0000000000000000\n",
      "类别 5: 1.0000000000000000\n",
      "\n",
      "混淆矩阵:\n",
      "[[9940    0    0    0    0    0]\n",
      " [   0 2044    0    0    0    0]\n",
      " [   0    0 2056    0    0    0]\n",
      " [   0    0    0 1927    0    0]\n",
      " [   0    0    0    0 1987    0]\n",
      " [   0    0    0    0    0 2043]]\n",
      "\n",
      "==================== 五折交叉验证总结 ====================\n",
      "准确率: 0.9999699969997000 ± 0.0000400040004000\n",
      "精确率: 0.9999700039741430 ± 0.0000399930314693\n",
      "召回率: 0.9999699969997000 ± 0.0000400040004000\n",
      "F1分数: 0.9999699926244052 ± 0.0000400103622810\n",
      "\n",
      "各类别的平均F1分数:\n",
      "类别 0: 0.9999800127884700 ± 0.0000244814803717\n",
      "类别 1: 0.9998989148478266 ± 0.0001238280245631\n",
      "类别 2: 1.0000000000000000 ± 0.0000000000000000\n",
      "类别 3: 0.9999032882011605 ± 0.0001934235976789\n",
      "类别 4: 1.0000000000000000 ± 0.0000000000000000\n",
      "类别 5: 1.0000000000000000 ± 0.0000000000000000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# \n",
    "accuracy_scores = []\n",
    "precision_scores = []\n",
    "recall_scores = []\n",
    "f1_scores = []\n",
    "f1_scores_per_class = []\n",
    "\n",
    "# \n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# \n",
    "for fold, (train_index, val_index) in enumerate(kf.split(X), 1):\n",
    "    print(f\"\\n{'='*20}  {fold} Folding verification {'='*20}\")\n",
    "    \n",
    "    # \n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "    \n",
    "    # \n",
    "    X_train, y_train = smote.fit_resample(X_train, y_train.astype('int'))\n",
    "    \n",
    "    print(f\"Training set size: {X_train.shape[0]}, Verification set size: {X_val.shape[0]}\")\n",
    "    \n",
    "    # \n",
    "    base_classifiers = [\n",
    "        ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "        ('cb', cbt.CatBoostClassifier(verbose=0,boosting_type='Plain')),\n",
    "        ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "    ]\n",
    "    blending_clf = VotingClassifier(estimators=base_classifiers, weights=None, voting='soft')\n",
    "    blending_clf.fit(X_train, y_train)\n",
    "    \n",
    "    # \n",
    "    base_classifiers_stack = [\n",
    "        ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),\n",
    "        ('et', ExtraTreesClassifier(n_estimators=100, random_state=42)),\n",
    "        ('xg', xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss'))\n",
    "    ]\n",
    "    meta_classifier = cbt.CatBoostClassifier(n_estimators=100, random_state=42, verbose=0)\n",
    "    stacking_clf = StackingClassifier(estimators=base_classifiers_stack, final_estimator=meta_classifier)\n",
    "    stacking_clf.fit(X_train, y_train)\n",
    "    \n",
    "    # \n",
    "    yt, yp = BEPCD(X_val, y_val, m1=blending_clf, m2=stacking_clf)\n",
    "    \n",
    "    # \n",
    "    current_accuracy = accuracy_score(yt, yp)\n",
    "    current_precision = precision_score(yt, yp, average='weighted')\n",
    "    current_recall = recall_score(yt, yp, average='weighted')\n",
    "    current_f1 = f1_score(yt, yp, average='weighted')\n",
    "    current_f1_per_class = f1_score(yt, yp, average=None)\n",
    "    \n",
    "    # \n",
    "    accuracy_scores.append(current_accuracy)\n",
    "    precision_scores.append(current_precision)\n",
    "    recall_scores.append(current_recall)\n",
    "    f1_scores.append(current_f1)\n",
    "    f1_scores_per_class.append(current_f1_per_class)\n",
    "    \n",
    "    # 输出当前折的详细结果\n",
    "    print(f\"\\nThe current assessment result:\")\n",
    "    print(f\"Accyuracy: {current_accuracy:.16f}\")\n",
    "    print(f\"精确率: {current_precision:.16f}\")\n",
    "    print(f\"召回率: {current_recall:.16f}\")\n",
    "    print(f\"F1分数: {current_f1:.16f}\")\n",
    "    \n",
    "    print(\"\\n各类别的F1分数:\")\n",
    "    for i, f1 in enumerate(current_f1_per_class):\n",
    "        print(f\"类别 {i}: {f1:.16f}\")\n",
    "    \n",
    "    # 输出混淆矩阵\n",
    "    print(\"\\n混淆矩阵:\")\n",
    "    cm = confusion_matrix(yt, yp)\n",
    "    print(cm)\n",
    "\n",
    "# 输出总体结果\n",
    "print(f\"\\n{'='*20} 五折交叉验证总结 {'='*20}\")\n",
    "print(f\"准确率: {np.mean(accuracy_scores):.16f} ± {np.std(accuracy_scores):.16f}\")\n",
    "print(f\"精确率: {np.mean(precision_scores):.16f} ± {np.std(precision_scores):.16f}\")\n",
    "print(f\"召回率: {np.mean(recall_scores):.16f} ± {np.std(recall_scores):.16f}\")\n",
    "print(f\"F1分数: {np.mean(f1_scores):.16f} ± {np.std(f1_scores):.16f}\")\n",
    "\n",
    "print(\"\\n各类别的平均F1分数:\")\n",
    "f1_per_class_mean = np.mean(f1_scores_per_class, axis=0)\n",
    "f1_per_class_std = np.std(f1_scores_per_class, axis=0)\n",
    "for i, (mean, std) in enumerate(zip(f1_per_class_mean, f1_per_class_std)):\n",
    "    print(f\"类别 {i}: {mean:.16f} ± {std:.16f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "3b4f55fe-7ac4-432a-8312-9b02a972e856",
   "metadata": {},
   "outputs": [],
   "source": [
    "def BEPCD(X_test, y_test, m1, m2,):\n",
    "    i = 0\n",
    "    t = []\n",
    "    m = []\n",
    "    yt = []\n",
    "    yp = []\n",
    "    l = []\n",
    "    pred_l = []\n",
    "    pro_l = []\n",
    "\n",
    "    # For each class (normal or a type of attack), find the PIONEER model\n",
    "    for xi, yi in stream.iter_pandas(X_test, y_test):\n",
    "\n",
    "        xi2=np.array(list(xi.values()))\n",
    "        y_pred1 = m1.predict(xi2.reshape(1, -1))      \n",
    "        y_pred1 = int(y_pred1[0])\n",
    "        y_pred2 = m2.predict(xi2.reshape(1, -1))      \n",
    "        y_pred2 = int(y_pred2[0])\n",
    "\n",
    "\n",
    "        p1 = m1.predict_proba(xi2.reshape(1, -1))      \n",
    "        p2 = m2.predict_proba(xi2.reshape(1, -1))     \n",
    "\n",
    "\n",
    "        # Find the highest prediction probability among all classes for each ML model\n",
    "        y_pred_p1 = np.max(p1)\n",
    "        y_pred_p2 = np.max(p2)\n",
    "\n",
    "        if y_pred1 == y_pred2: \n",
    "            y_pred = y_pred1 \n",
    "\n",
    "        elif y_pred1 != y_pred2: # If the predicted classes of all the  models are different\n",
    "            # For each prediction model, check if the predicted class’s original ML model is the same as its PIONEER model\n",
    "            if model[y_pred1]==m1 and model[y_pred2]!=m2: \n",
    "                l.append(m1)\n",
    "                pred_l.append(y_pred1) # Save the predicted class\n",
    "                pro_l.append(y_pred_p1) # Save the confidence\n",
    "\n",
    "            if model[y_pred2]==m2 and model[y_pred1]!=m1: \n",
    "                l.append(m2)\n",
    "                pred_l.append(y_pred2)\n",
    "                pro_l.append(y_pred_p2)\n",
    "\n",
    "            if len(l)==0: # Avoid empty probability list\n",
    "                pro_l=[y_pred_p1,y_pred_p2]              \n",
    "            else:\n",
    "                max_p = max(pro_l)\n",
    "                \n",
    "                if max_p == y_pred_p1:\n",
    "                    y_pred = y_pred1\n",
    "                else:\n",
    "                    y_pred = y_pred2  \n",
    "        \n",
    "\n",
    "        yt.append(yi)\n",
    "        yp.append(y_pred) \n",
    "    return yt, yp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "0a905a29-aed9-4821-ad96-77f8f28f6319",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3h 35min 30s\n",
      "Wall time: 13min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "yt, yp = BEPCD(X_test, y_test, m1 = blending_clf, m2 = stacking_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ca645f03-7043-4f6c-8975-edce61ccc182",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of BEPCD: 0.9998999899989999\n",
      "Precision of BEPCD: 0.9999000100491203\n",
      "Recall of BEPCD: 0.9998999899989999\n",
      "F1-score of BEPCD: 0.9998999697569818\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      9974\n",
      "           1       1.00      1.00      1.00      2025\n",
      "           2       1.00      1.00      1.00      2011\n",
      "           3       1.00      1.00      1.00      1981\n",
      "           4       1.00      1.00      1.00      2032\n",
      "           5       1.00      1.00      1.00      1975\n",
      "\n",
      "    accuracy                           1.00     19998\n",
      "   macro avg       1.00      1.00      1.00     19998\n",
      "weighted avg       1.00      1.00      1.00     19998\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 26112 characters]",
      "text/plain": [
       "<Figure size 500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# The performance of the proposed BEPCD model\n",
    "\n",
    "print('Accuracy of BEPCD: '+ str(rf_score))\n",
    "precision,recall,fscore,none= precision_recall_fscore_support(y_true, y_predict, average='weighted') \n",
    "print('Precision of BEPCD: '+(str(precision)))\n",
    "print('Recall of BEPCD: '+(str(recall)))\n",
    "print('F1-score of BEPCD: '+(str(fscore)))\n",
    "print(classification_report(y_true,y_predict))\n",
    "cm=confusion_matrix(y_true,y_predict)\n",
    "f,ax=plt.subplots(figsize=(5,5))\n",
    "sns.heatmap(cm,annot=True,linewidth=0.5,linecolor=\"red\",fmt=\".0f\",ax=ax)\n",
    "plt.xlabel(\"y_pred\")\n",
    "plt.ylabel(\"y_true\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "835a3c69-627b-457b-a31c-4768737a6e49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Class 0: 1.00000\n",
      "Accuracy of Class 1: 1.00000\n",
      "Accuracy of Class 2: 1.00000\n",
      "Accuracy of Class 3: 0.99899\n",
      "Accuracy of Class 4: 1.00000\n",
      "Accuracy of Class 5: 1.00000\n",
      "Class 0 - Precision: 0.99980, Recall: 1.00000, F1 Score: 0.99990\n",
      "Class 1 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n",
      "Class 2 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n",
      "Class 3 - Precision: 1.00000, Recall: 0.99899, F1 Score: 0.99949\n",
      "Class 4 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n",
      "Class 5 - Precision: 1.00000, Recall: 1.00000, F1 Score: 1.00000\n"
     ]
    }
   ],
   "source": [
    "class_accuracies = cm.diagonal() / cm.sum(axis=1)\n",
    "class_names = ['Class {}'.format(i) for i in range(cm.shape[0])]\n",
    "for name, accuracy in zip(class_names, class_accuracies):\n",
    "    print(f'Accuracy of {name}: {accuracy:.5f}')\n",
    "precision, recall, fscore, _ = precision_recall_fscore_support(y_true, y_predict, average=None, zero_division=0)\n",
    "for i, (p, r, f) in enumerate(zip(precision, recall, fscore)):\n",
    "    print(f'Class {i} - Precision: {p:.5f}, Recall: {r:.5f}, F1 Score: {f:.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b006596-7ffa-4080-b01c-c2f45d2ed14f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
