{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "482b1a8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Height</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Total BMD</th>\n",
       "      <th>Lumbar spine BMD</th>\n",
       "      <th>TBLH BMD</th>\n",
       "      <th>TBLH BMD Z -score</th>\n",
       "      <th>Total fat percentage</th>\n",
       "      <th>...</th>\n",
       "      <th>LDL</th>\n",
       "      <th>TC</th>\n",
       "      <th>FBG</th>\n",
       "      <th>TyG index</th>\n",
       "      <th>HOMA-IR</th>\n",
       "      <th>HbA1c</th>\n",
       "      <th>Spexin</th>\n",
       "      <th>FGF23</th>\n",
       "      <th>1,25(OH)2D</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>49.0</td>\n",
       "      <td>162.9</td>\n",
       "      <td>18.47</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.88</td>\n",
       "      <td>1.07</td>\n",
       "      <td>28.4</td>\n",
       "      <td>...</td>\n",
       "      <td>2.44</td>\n",
       "      <td>2.87</td>\n",
       "      <td>5.20</td>\n",
       "      <td>8.72</td>\n",
       "      <td>2.35</td>\n",
       "      <td>5.27</td>\n",
       "      <td>0.52</td>\n",
       "      <td>46.11</td>\n",
       "      <td>65.30</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>41.3</td>\n",
       "      <td>161.5</td>\n",
       "      <td>15.83</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.80</td>\n",
       "      <td>1.42</td>\n",
       "      <td>28.6</td>\n",
       "      <td>...</td>\n",
       "      <td>1.40</td>\n",
       "      <td>2.68</td>\n",
       "      <td>5.14</td>\n",
       "      <td>8.23</td>\n",
       "      <td>3.05</td>\n",
       "      <td>4.59</td>\n",
       "      <td>0.52</td>\n",
       "      <td>32.61</td>\n",
       "      <td>57.26</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>53.0</td>\n",
       "      <td>161.1</td>\n",
       "      <td>20.42</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.85</td>\n",
       "      <td>1.92</td>\n",
       "      <td>35.7</td>\n",
       "      <td>...</td>\n",
       "      <td>2.18</td>\n",
       "      <td>3.86</td>\n",
       "      <td>4.56</td>\n",
       "      <td>8.34</td>\n",
       "      <td>2.15</td>\n",
       "      <td>4.25</td>\n",
       "      <td>0.68</td>\n",
       "      <td>69.63</td>\n",
       "      <td>65.90</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>50.5</td>\n",
       "      <td>161.0</td>\n",
       "      <td>19.48</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.70</td>\n",
       "      <td>1.35</td>\n",
       "      <td>26.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.31</td>\n",
       "      <td>2.72</td>\n",
       "      <td>4.87</td>\n",
       "      <td>8.53</td>\n",
       "      <td>2.87</td>\n",
       "      <td>5.16</td>\n",
       "      <td>0.55</td>\n",
       "      <td>44.47</td>\n",
       "      <td>92.20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>47.2</td>\n",
       "      <td>159.4</td>\n",
       "      <td>18.58</td>\n",
       "      <td>1.05</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.82</td>\n",
       "      <td>24.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.30</td>\n",
       "      <td>3.29</td>\n",
       "      <td>5.33</td>\n",
       "      <td>8.63</td>\n",
       "      <td>2.15</td>\n",
       "      <td>4.22</td>\n",
       "      <td>0.63</td>\n",
       "      <td>39.76</td>\n",
       "      <td>25.60</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender  Age  Weight  Height    BMI  Total BMD  Lumbar spine BMD  TBLH BMD  \\\n",
       "0       0   11    49.0   162.9  18.47       0.97              0.74      0.88   \n",
       "1       0   10    41.3   161.5  15.83       0.90              0.64      0.80   \n",
       "2       0   11    53.0   161.1  20.42       0.97              0.72      0.85   \n",
       "3       1   11    50.5   161.0  19.48       0.75              0.79      0.70   \n",
       "4       0   11    47.2   159.4  18.58       1.05              0.82      0.92   \n",
       "\n",
       "   TBLH BMD Z -score  Total fat percentage   ...   LDL    TC   FBG  TyG index  \\\n",
       "0               1.07                   28.4  ...  2.44  2.87  5.20       8.72   \n",
       "1               1.42                   28.6  ...  1.40  2.68  5.14       8.23   \n",
       "2               1.92                   35.7  ...  2.18  3.86  4.56       8.34   \n",
       "3               1.35                   26.0  ...  2.31  2.72  4.87       8.53   \n",
       "4               0.82                   24.0  ...  2.30  3.29  5.33       8.63   \n",
       "\n",
       "   HOMA-IR  HbA1c  Spexin  FGF23  1,25(OH)2D  target  \n",
       "0     2.35   5.27    0.52  46.11       65.30       1  \n",
       "1     3.05   4.59    0.52  32.61       57.26       0  \n",
       "2     2.15   4.25    0.68  69.63       65.90       1  \n",
       "3     2.87   5.16    0.55  44.47       92.20       1  \n",
       "4     2.15   4.22    0.63  39.76       25.60       1  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np # 导入NumPy数学工具箱\n",
    "import pandas as pd # 导入Pandas数据处理工具箱\n",
    "from sklearn.metrics import roc_auc_score,roc_curve,auc\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "df = pd.read_csv(\"G:/儿童骨密度/DATA/Raw data/p_NMID_2.csv\")  # 读取文件\n",
    "df.head() # 显示前5行数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5058aee7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13408 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns #导入seaborn画图工具箱\n",
    "sns.countplot(x=\"target\", data=df, palette=\"bwr\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f1cb215b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Height</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Total BMD</th>\n",
       "      <th>Lumbar spine BMD</th>\n",
       "      <th>TBLH BMD</th>\n",
       "      <th>TBLH BMD Z -score</th>\n",
       "      <th>Total fat percentage</th>\n",
       "      <th>...</th>\n",
       "      <th>LDL</th>\n",
       "      <th>TC</th>\n",
       "      <th>FBG</th>\n",
       "      <th>TyG index</th>\n",
       "      <th>HOMA-IR</th>\n",
       "      <th>HbA1c</th>\n",
       "      <th>Spexin</th>\n",
       "      <th>FGF23</th>\n",
       "      <th>1,25(OH)2D</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>49.0</td>\n",
       "      <td>162.9</td>\n",
       "      <td>18.47</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.88</td>\n",
       "      <td>1.07</td>\n",
       "      <td>28.4</td>\n",
       "      <td>...</td>\n",
       "      <td>2.44</td>\n",
       "      <td>2.87</td>\n",
       "      <td>5.20</td>\n",
       "      <td>8.72</td>\n",
       "      <td>2.35</td>\n",
       "      <td>5.27</td>\n",
       "      <td>0.52</td>\n",
       "      <td>46.11</td>\n",
       "      <td>65.30</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>41.3</td>\n",
       "      <td>161.5</td>\n",
       "      <td>15.83</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.80</td>\n",
       "      <td>1.42</td>\n",
       "      <td>28.6</td>\n",
       "      <td>...</td>\n",
       "      <td>1.40</td>\n",
       "      <td>2.68</td>\n",
       "      <td>5.14</td>\n",
       "      <td>8.23</td>\n",
       "      <td>3.05</td>\n",
       "      <td>4.59</td>\n",
       "      <td>0.52</td>\n",
       "      <td>32.61</td>\n",
       "      <td>57.26</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>53.0</td>\n",
       "      <td>161.1</td>\n",
       "      <td>20.42</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.85</td>\n",
       "      <td>1.92</td>\n",
       "      <td>35.7</td>\n",
       "      <td>...</td>\n",
       "      <td>2.18</td>\n",
       "      <td>3.86</td>\n",
       "      <td>4.56</td>\n",
       "      <td>8.34</td>\n",
       "      <td>2.15</td>\n",
       "      <td>4.25</td>\n",
       "      <td>0.68</td>\n",
       "      <td>69.63</td>\n",
       "      <td>65.90</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>50.5</td>\n",
       "      <td>161.0</td>\n",
       "      <td>19.48</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.70</td>\n",
       "      <td>1.35</td>\n",
       "      <td>26.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.31</td>\n",
       "      <td>2.72</td>\n",
       "      <td>4.87</td>\n",
       "      <td>8.53</td>\n",
       "      <td>2.87</td>\n",
       "      <td>5.16</td>\n",
       "      <td>0.55</td>\n",
       "      <td>44.47</td>\n",
       "      <td>92.20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>47.2</td>\n",
       "      <td>159.4</td>\n",
       "      <td>18.58</td>\n",
       "      <td>1.05</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.82</td>\n",
       "      <td>24.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.30</td>\n",
       "      <td>3.29</td>\n",
       "      <td>5.33</td>\n",
       "      <td>8.63</td>\n",
       "      <td>2.15</td>\n",
       "      <td>4.22</td>\n",
       "      <td>0.63</td>\n",
       "      <td>39.76</td>\n",
       "      <td>25.60</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender  Age  Weight  Height    BMI  Total BMD  Lumbar spine BMD  TBLH BMD  \\\n",
       "0       0   11    49.0   162.9  18.47       0.97              0.74      0.88   \n",
       "1       0   10    41.3   161.5  15.83       0.90              0.64      0.80   \n",
       "2       0   11    53.0   161.1  20.42       0.97              0.72      0.85   \n",
       "3       1   11    50.5   161.0  19.48       0.75              0.79      0.70   \n",
       "4       0   11    47.2   159.4  18.58       1.05              0.82      0.92   \n",
       "\n",
       "   TBLH BMD Z -score  Total fat percentage   ...   LDL    TC   FBG  TyG index  \\\n",
       "0               1.07                   28.4  ...  2.44  2.87  5.20       8.72   \n",
       "1               1.42                   28.6  ...  1.40  2.68  5.14       8.23   \n",
       "2               1.92                   35.7  ...  2.18  3.86  4.56       8.34   \n",
       "3               1.35                   26.0  ...  2.31  2.72  4.87       8.53   \n",
       "4               0.82                   24.0  ...  2.30  3.29  5.33       8.63   \n",
       "\n",
       "   HOMA-IR  HbA1c  Spexin  FGF23  1,25(OH)2D  target  \n",
       "0     2.35   5.27    0.52  46.11       65.30       1  \n",
       "1     3.05   4.59    0.52  32.61       57.26       0  \n",
       "2     2.15   4.25    0.68  69.63       65.90       1  \n",
       "3     2.87   5.16    0.55  44.47       92.20       1  \n",
       "4     2.15   4.22    0.63  39.76       25.60       1  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将某些特征转换为数值类型的哑变量\n",
    "#a = pd.get_dummies(df['cp'], prefix = \"cp\")\n",
    "#b = pd.get_dummies(df['thal'], prefix = \"thal\")\n",
    "#c = pd.get_dummies(df['slope'], prefix = \"slope\")\n",
    "#frames = [df, a, b, c]\n",
    "#df = pd.concat(frames, axis = 1)\n",
    "#df.head()\n",
    "#df = df.drop(columns = ['ID'])#去除ID列\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "97148168",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 2734528 characters]",
      "text/plain": [
       "<Figure size 5400x5400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#特征相关性热图\n",
    "# 导入数据可视化所需要的库\n",
    "import matplotlib.pyplot as plt #Matplotlib为 Python画图工具库\n",
    "import seaborn as sns #Seaborn为统计学数据可视化工具库\n",
    "#增加图片清晰度\n",
    "\n",
    "#对所有的标签和特征两两显示其相关性的热力图\n",
    "plt.figure(figsize=(6, 6), dpi=900)\n",
    "sns.heatmap(df.corr(),cmap='coolwarm',annot=True , fmt='.2f', linewidths=.5,annot_kws={'size': 4})\n",
    "\n",
    "plt.show() #plt代表英文 plot, 就是画图的意思\n",
    "#1. df_heart: 需要展示的数据，可以是DataFrame或numpy数组\n",
    "#2. cmap: 颜色映射表，设置热图的颜色主题。它可以是内置颜色映射表（如'coolwarm'、'viridis'、'mako'、'rocket'、'copper'等），也可以是自定义的颜色列表\n",
    "#3. annot: 是否显示每个单元格的数值\n",
    "#4. fmt: 数值格式，用于控制annot参数输出数值的格式,.2f保留2位小数\n",
    "#5. linewidths: 热图中每个矩形之间的白线间隔线宽度,annot_kws={'size': 6}设置字体大小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f193102e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "张量 X的形状 : (332, 22)\n",
      "张量 y的形状 : (332, 1)\n"
     ]
    }
   ],
   "source": [
    "#2.构建特征集和标签集\n",
    "X = df.drop(['target'], axis = 1) # 构建特征集\n",
    "y = df.target.values # 构建标签集\n",
    "y = y.reshape(-1, 1) # -1是相对索引 , 等价于 len(y)\n",
    "print(\"张量 X的形状 :\", X.shape)\n",
    "print(\"张量 y的形状 :\", y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "938f12db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split # 拆分训练集和测试集,\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.2,random_state=55)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "483eebeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler =StandardScaler() \n",
    "X_train = scaler.ﬁt_transform(X_train) \n",
    "X_test = scaler.transform(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fba79db2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM混淆矩阵:\n",
      " [[34 12]\n",
      " [ 0 21]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\Lib\\site-packages\\sklearn\\utils\\validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'SVM(Best Score)Confusion Matrix'}>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 211016 characters]",
      "text/plain": [
       "<Figure size 3600x3600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.svm import SVC # 导入SVM分类器\n",
    "from sklearn.metrics import classiﬁcation_report # 导入分类报告\n",
    "from sklearn.metrics import confusion_matrix,precision_score,recall_score,f1_score\n",
    "from sklearn.metrics import (f1_score, confusion_matrix) # 导入评估指标\n",
    "svm = SVC(random_state = 0,probability=True)\n",
    "svm.fit(X_train, y_train)\n",
    "y_pred1 = svm.predict(X_test) # \n",
    "svm_acc = svm.score(X_test,y_test)*100\n",
    "y_prob1 = svm.predict_proba(X_test)[:,1]\n",
    "cm_dot_four = confusion_matrix(y_prob1 >= 0.55, y_pred1)\n",
    "plt.figure(figsize=(4, 4), dpi=900)#设置图片大小，像素\n",
    "print('SVM混淆矩阵:\\n', cm_dot_four)\n",
    "plt.title(\"SVM(Best Score)Confusion Matrix\")#KNN（最优参数）混淆矩阵\n",
    "sns.heatmap(cm_dot_four, annot=True, cmap=\"coolwarm\", fmt=\"d\", cbar=\"TRUE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d127484d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original ROC area: 0.668\n",
      "Bootstrap #1 ROC area: 0.630\n",
      "Bootstrap #2 ROC area: 0.563\n",
      "Bootstrap #3 ROC area: 0.613\n",
      "Bootstrap #4 ROC area: 0.649\n",
      "Bootstrap #5 ROC area: 0.779\n",
      "Bootstrap #6 ROC area: 0.703\n",
      "Bootstrap #7 ROC area: 0.546\n",
      "Bootstrap #8 ROC area: 0.654\n",
      "Bootstrap #9 ROC area: 0.771\n",
      "Bootstrap #10 ROC area: 0.738\n",
      "Bootstrap #11 ROC area: 0.535\n",
      "Bootstrap #12 ROC area: 0.645\n",
      "Bootstrap #13 ROC area: 0.646\n",
      "Bootstrap #14 ROC area: 0.743\n",
      "Bootstrap #15 ROC area: 0.712\n",
      "Bootstrap #16 ROC area: 0.596\n",
      "Bootstrap #17 ROC area: 0.726\n",
      "Bootstrap #18 ROC area: 0.768\n",
      "Bootstrap #19 ROC area: 0.617\n",
      "Bootstrap #20 ROC area: 0.622\n",
      "Bootstrap #21 ROC area: 0.653\n",
      "Bootstrap #22 ROC area: 0.621\n",
      "Bootstrap #23 ROC area: 0.660\n",
      "Bootstrap #24 ROC area: 0.785\n",
      "Bootstrap #25 ROC area: 0.584\n",
      "Bootstrap #26 ROC area: 0.595\n",
      "Bootstrap #27 ROC area: 0.688\n",
      "Bootstrap #28 ROC area: 0.639\n",
      "Bootstrap #29 ROC area: 0.538\n",
      "Bootstrap #30 ROC area: 0.572\n",
      "Bootstrap #31 ROC area: 0.750\n",
      "Bootstrap #32 ROC area: 0.685\n",
      "Bootstrap #33 ROC area: 0.632\n",
      "Bootstrap #34 ROC area: 0.635\n",
      "Bootstrap #35 ROC area: 0.696\n",
      "Bootstrap #36 ROC area: 0.677\n",
      "Bootstrap #37 ROC area: 0.680\n",
      "Bootstrap #38 ROC area: 0.650\n",
      "Bootstrap #39 ROC area: 0.669\n",
      "Bootstrap #40 ROC area: 0.641\n",
      "Bootstrap #41 ROC area: 0.701\n",
      "Bootstrap #42 ROC area: 0.591\n",
      "Bootstrap #43 ROC area: 0.540\n",
      "Bootstrap #44 ROC area: 0.547\n",
      "Bootstrap #45 ROC area: 0.683\n",
      "Bootstrap #46 ROC area: 0.595\n",
      "Bootstrap #47 ROC area: 0.623\n",
      "Bootstrap #48 ROC area: 0.644\n",
      "Bootstrap #49 ROC area: 0.755\n",
      "Bootstrap #50 ROC area: 0.625\n",
      "Bootstrap #51 ROC area: 0.691\n",
      "Bootstrap #52 ROC area: 0.754\n",
      "Bootstrap #53 ROC area: 0.668\n",
      "Bootstrap #54 ROC area: 0.551\n",
      "Bootstrap #55 ROC area: 0.700\n",
      "Bootstrap #56 ROC area: 0.656\n",
      "Bootstrap #57 ROC area: 0.609\n",
      "Bootstrap #58 ROC area: 0.648\n",
      "Bootstrap #59 ROC area: 0.522\n",
      "Bootstrap #60 ROC area: 0.709\n",
      "Bootstrap #61 ROC area: 0.652\n",
      "Bootstrap #62 ROC area: 0.668\n",
      "Bootstrap #63 ROC area: 0.667\n",
      "Bootstrap #64 ROC area: 0.639\n",
      "Bootstrap #65 ROC area: 0.726\n",
      "Bootstrap #66 ROC area: 0.558\n",
      "Bootstrap #67 ROC area: 0.761\n",
      "Bootstrap #68 ROC area: 0.770\n",
      "Bootstrap #69 ROC area: 0.612\n",
      "Bootstrap #70 ROC area: 0.560\n",
      "Bootstrap #71 ROC area: 0.687\n",
      "Bootstrap #72 ROC area: 0.611\n",
      "Bootstrap #73 ROC area: 0.739\n",
      "Bootstrap #74 ROC area: 0.641\n",
      "Bootstrap #75 ROC area: 0.608\n",
      "Bootstrap #76 ROC area: 0.785\n",
      "Bootstrap #77 ROC area: 0.696\n",
      "Bootstrap #78 ROC area: 0.608\n",
      "Bootstrap #79 ROC area: 0.831\n",
      "Bootstrap #80 ROC area: 0.496\n",
      "Bootstrap #81 ROC area: 0.615\n",
      "Bootstrap #82 ROC area: 0.697\n",
      "Bootstrap #83 ROC area: 0.594\n",
      "Bootstrap #84 ROC area: 0.639\n",
      "Bootstrap #85 ROC area: 0.639\n",
      "Bootstrap #86 ROC area: 0.681\n",
      "Bootstrap #87 ROC area: 0.634\n",
      "Bootstrap #88 ROC area: 0.643\n",
      "Bootstrap #89 ROC area: 0.632\n",
      "Bootstrap #90 ROC area: 0.725\n",
      "Bootstrap #91 ROC area: 0.609\n",
      "Bootstrap #92 ROC area: 0.703\n",
      "Bootstrap #93 ROC area: 0.797\n",
      "Bootstrap #94 ROC area: 0.709\n",
      "Bootstrap #95 ROC area: 0.696\n",
      "Bootstrap #96 ROC area: 0.718\n",
      "Bootstrap #97 ROC area: 0.700\n",
      "Bootstrap #98 ROC area: 0.564\n",
      "Bootstrap #99 ROC area: 0.686\n",
      "Bootstrap #100 ROC area: 0.684\n",
      "Bootstrap #101 ROC area: 0.574\n",
      "Bootstrap #102 ROC area: 0.719\n",
      "Bootstrap #103 ROC area: 0.698\n",
      "Bootstrap #104 ROC area: 0.645\n",
      "Bootstrap #105 ROC area: 0.756\n",
      "Bootstrap #106 ROC area: 0.680\n",
      "Bootstrap #107 ROC area: 0.585\n",
      "Bootstrap #108 ROC area: 0.724\n",
      "Bootstrap #109 ROC area: 0.706\n",
      "Bootstrap #110 ROC area: 0.747\n",
      "Bootstrap #111 ROC area: 0.631\n",
      "Bootstrap #112 ROC area: 0.689\n",
      "Bootstrap #113 ROC area: 0.524\n",
      "Bootstrap #114 ROC area: 0.675\n",
      "Bootstrap #115 ROC area: 0.755\n",
      "Bootstrap #116 ROC area: 0.649\n",
      "Bootstrap #117 ROC area: 0.698\n",
      "Bootstrap #118 ROC area: 0.864\n",
      "Bootstrap #119 ROC area: 0.548\n",
      "Bootstrap #120 ROC area: 0.682\n",
      "Bootstrap #121 ROC area: 0.761\n",
      "Bootstrap #122 ROC area: 0.672\n",
      "Bootstrap #123 ROC area: 0.792\n",
      "Bootstrap #124 ROC area: 0.433\n",
      "Bootstrap #125 ROC area: 0.799\n",
      "Bootstrap #126 ROC area: 0.699\n",
      "Bootstrap #127 ROC area: 0.673\n",
      "Bootstrap #128 ROC area: 0.696\n",
      "Bootstrap #129 ROC area: 0.558\n",
      "Bootstrap #130 ROC area: 0.633\n",
      "Bootstrap #131 ROC area: 0.683\n",
      "Bootstrap #132 ROC area: 0.613\n",
      "Bootstrap #133 ROC area: 0.547\n",
      "Bootstrap #134 ROC area: 0.549\n",
      "Bootstrap #135 ROC area: 0.697\n",
      "Bootstrap #136 ROC area: 0.678\n",
      "Bootstrap #137 ROC area: 0.612\n",
      "Bootstrap #138 ROC area: 0.639\n",
      "Bootstrap #139 ROC area: 0.675\n",
      "Bootstrap #140 ROC area: 0.639\n",
      "Bootstrap #141 ROC area: 0.487\n",
      "Bootstrap #142 ROC area: 0.680\n",
      "Bootstrap #143 ROC area: 0.741\n",
      "Bootstrap #144 ROC area: 0.693\n",
      "Bootstrap #145 ROC area: 0.640\n",
      "Bootstrap #146 ROC area: 0.646\n",
      "Bootstrap #147 ROC area: 0.692\n",
      "Bootstrap #148 ROC area: 0.742\n",
      "Bootstrap #149 ROC area: 0.655\n",
      "Bootstrap #150 ROC area: 0.751\n",
      "Bootstrap #151 ROC area: 0.700\n",
      "Bootstrap #152 ROC area: 0.695\n",
      "Bootstrap #153 ROC area: 0.597\n",
      "Bootstrap #154 ROC area: 0.578\n",
      "Bootstrap #155 ROC area: 0.672\n",
      "Bootstrap #156 ROC area: 0.619\n",
      "Bootstrap #157 ROC area: 0.636\n",
      "Bootstrap #158 ROC area: 0.766\n",
      "Bootstrap #159 ROC area: 0.659\n",
      "Bootstrap #160 ROC area: 0.673\n",
      "Bootstrap #161 ROC area: 0.630\n",
      "Bootstrap #162 ROC area: 0.642\n",
      "Bootstrap #163 ROC area: 0.592\n",
      "Bootstrap #164 ROC area: 0.618\n",
      "Bootstrap #165 ROC area: 0.671\n",
      "Bootstrap #166 ROC area: 0.602\n",
      "Bootstrap #167 ROC area: 0.597\n",
      "Bootstrap #168 ROC area: 0.719\n",
      "Bootstrap #169 ROC area: 0.655\n",
      "Bootstrap #170 ROC area: 0.719\n",
      "Bootstrap #171 ROC area: 0.853\n",
      "Bootstrap #172 ROC area: 0.544\n",
      "Bootstrap #173 ROC area: 0.724\n",
      "Bootstrap #174 ROC area: 0.500\n",
      "Bootstrap #175 ROC area: 0.661\n",
      "Bootstrap #176 ROC area: 0.706\n",
      "Bootstrap #177 ROC area: 0.591\n",
      "Bootstrap #178 ROC area: 0.556\n",
      "Bootstrap #179 ROC area: 0.718\n",
      "Bootstrap #180 ROC area: 0.595\n",
      "Bootstrap #181 ROC area: 0.699\n",
      "Bootstrap #182 ROC area: 0.700\n",
      "Bootstrap #183 ROC area: 0.551\n",
      "Bootstrap #184 ROC area: 0.654\n",
      "Bootstrap #185 ROC area: 0.689\n",
      "Bootstrap #186 ROC area: 0.786\n",
      "Bootstrap #187 ROC area: 0.714\n",
      "Bootstrap #188 ROC area: 0.694\n",
      "Bootstrap #189 ROC area: 0.674\n",
      "Bootstrap #190 ROC area: 0.631\n",
      "Bootstrap #191 ROC area: 0.546\n",
      "Bootstrap #192 ROC area: 0.732\n",
      "Bootstrap #193 ROC area: 0.730\n",
      "Bootstrap #194 ROC area: 0.642\n",
      "Bootstrap #195 ROC area: 0.627\n",
      "Bootstrap #196 ROC area: 0.628\n",
      "Bootstrap #197 ROC area: 0.863\n",
      "Bootstrap #198 ROC area: 0.709\n",
      "Bootstrap #199 ROC area: 0.551\n",
      "Bootstrap #200 ROC area: 0.587\n",
      "Bootstrap #201 ROC area: 0.638\n",
      "Bootstrap #202 ROC area: 0.665\n",
      "Bootstrap #203 ROC area: 0.562\n",
      "Bootstrap #204 ROC area: 0.693\n",
      "Bootstrap #205 ROC area: 0.694\n",
      "Bootstrap #206 ROC area: 0.712\n",
      "Bootstrap #207 ROC area: 0.706\n",
      "Bootstrap #208 ROC area: 0.673\n",
      "Bootstrap #209 ROC area: 0.710\n",
      "Bootstrap #210 ROC area: 0.716\n",
      "Bootstrap #211 ROC area: 0.509\n",
      "Bootstrap #212 ROC area: 0.517\n",
      "Bootstrap #213 ROC area: 0.658\n",
      "Bootstrap #214 ROC area: 0.626\n",
      "Bootstrap #215 ROC area: 0.724\n",
      "Bootstrap #216 ROC area: 0.740\n",
      "Bootstrap #217 ROC area: 0.589\n",
      "Bootstrap #218 ROC area: 0.614\n",
      "Bootstrap #219 ROC area: 0.701\n",
      "Bootstrap #220 ROC area: 0.678\n",
      "Bootstrap #221 ROC area: 0.723\n",
      "Bootstrap #222 ROC area: 0.554\n",
      "Bootstrap #223 ROC area: 0.623\n",
      "Bootstrap #224 ROC area: 0.589\n",
      "Bootstrap #225 ROC area: 0.735\n",
      "Bootstrap #226 ROC area: 0.712\n",
      "Bootstrap #227 ROC area: 0.780\n",
      "Bootstrap #228 ROC area: 0.730\n",
      "Bootstrap #229 ROC area: 0.683\n",
      "Bootstrap #230 ROC area: 0.650\n",
      "Bootstrap #231 ROC area: 0.715\n",
      "Bootstrap #232 ROC area: 0.740\n",
      "Bootstrap #233 ROC area: 0.613\n",
      "Bootstrap #234 ROC area: 0.701\n",
      "Bootstrap #235 ROC area: 0.598\n",
      "Bootstrap #236 ROC area: 0.569\n",
      "Bootstrap #237 ROC area: 0.645\n",
      "Bootstrap #238 ROC area: 0.605\n",
      "Bootstrap #239 ROC area: 0.666\n",
      "Bootstrap #240 ROC area: 0.680\n",
      "Bootstrap #241 ROC area: 0.655\n",
      "Bootstrap #242 ROC area: 0.655\n",
      "Bootstrap #243 ROC area: 0.643\n",
      "Bootstrap #244 ROC area: 0.615\n",
      "Bootstrap #245 ROC area: 0.640\n",
      "Bootstrap #246 ROC area: 0.687\n",
      "Bootstrap #247 ROC area: 0.610\n",
      "Bootstrap #248 ROC area: 0.692\n",
      "Bootstrap #249 ROC area: 0.660\n",
      "Bootstrap #250 ROC area: 0.714\n",
      "Bootstrap #251 ROC area: 0.613\n",
      "Bootstrap #252 ROC area: 0.754\n",
      "Bootstrap #253 ROC area: 0.642\n",
      "Bootstrap #254 ROC area: 0.637\n",
      "Bootstrap #255 ROC area: 0.562\n",
      "Bootstrap #256 ROC area: 0.624\n",
      "Bootstrap #257 ROC area: 0.654\n",
      "Bootstrap #258 ROC area: 0.711\n",
      "Bootstrap #259 ROC area: 0.569\n",
      "Bootstrap #260 ROC area: 0.775\n",
      "Bootstrap #261 ROC area: 0.671\n",
      "Bootstrap #262 ROC area: 0.733\n",
      "Bootstrap #263 ROC area: 0.710\n",
      "Bootstrap #264 ROC area: 0.682\n",
      "Bootstrap #265 ROC area: 0.618\n",
      "Bootstrap #266 ROC area: 0.704\n",
      "Bootstrap #267 ROC area: 0.582\n",
      "Bootstrap #268 ROC area: 0.700\n",
      "Bootstrap #269 ROC area: 0.762\n",
      "Bootstrap #270 ROC area: 0.659\n",
      "Bootstrap #271 ROC area: 0.713\n",
      "Bootstrap #272 ROC area: 0.709\n",
      "Bootstrap #273 ROC area: 0.690\n",
      "Bootstrap #274 ROC area: 0.694\n",
      "Bootstrap #275 ROC area: 0.749\n",
      "Bootstrap #276 ROC area: 0.821\n",
      "Bootstrap #277 ROC area: 0.651\n",
      "Bootstrap #278 ROC area: 0.758\n",
      "Bootstrap #279 ROC area: 0.693\n",
      "Bootstrap #280 ROC area: 0.579\n",
      "Bootstrap #281 ROC area: 0.684\n",
      "Bootstrap #282 ROC area: 0.633\n",
      "Bootstrap #283 ROC area: 0.707\n",
      "Bootstrap #284 ROC area: 0.763\n",
      "Bootstrap #285 ROC area: 0.721\n",
      "Bootstrap #286 ROC area: 0.649\n",
      "Bootstrap #287 ROC area: 0.559\n",
      "Bootstrap #288 ROC area: 0.679\n",
      "Bootstrap #289 ROC area: 0.542\n",
      "Bootstrap #290 ROC area: 0.503\n",
      "Bootstrap #291 ROC area: 0.703\n",
      "Bootstrap #292 ROC area: 0.791\n",
      "Bootstrap #293 ROC area: 0.720\n",
      "Bootstrap #294 ROC area: 0.695\n",
      "Bootstrap #295 ROC area: 0.632\n",
      "Bootstrap #296 ROC area: 0.655\n",
      "Bootstrap #297 ROC area: 0.678\n",
      "Bootstrap #298 ROC area: 0.568\n",
      "Bootstrap #299 ROC area: 0.812\n",
      "Bootstrap #300 ROC area: 0.640\n",
      "Bootstrap #301 ROC area: 0.651\n",
      "Bootstrap #302 ROC area: 0.674\n",
      "Bootstrap #303 ROC area: 0.685\n",
      "Bootstrap #304 ROC area: 0.681\n",
      "Bootstrap #305 ROC area: 0.661\n",
      "Bootstrap #306 ROC area: 0.599\n",
      "Bootstrap #307 ROC area: 0.692\n",
      "Bootstrap #308 ROC area: 0.651\n",
      "Bootstrap #309 ROC area: 0.733\n",
      "Bootstrap #310 ROC area: 0.627\n",
      "Bootstrap #311 ROC area: 0.575\n",
      "Bootstrap #312 ROC area: 0.674\n",
      "Bootstrap #313 ROC area: 0.598\n",
      "Bootstrap #314 ROC area: 0.696\n",
      "Bootstrap #315 ROC area: 0.675\n",
      "Bootstrap #316 ROC area: 0.773\n",
      "Bootstrap #317 ROC area: 0.714\n",
      "Bootstrap #318 ROC area: 0.712\n",
      "Bootstrap #319 ROC area: 0.582\n",
      "Bootstrap #320 ROC area: 0.639\n",
      "Bootstrap #321 ROC area: 0.605\n",
      "Bootstrap #322 ROC area: 0.563\n",
      "Bootstrap #323 ROC area: 0.692\n",
      "Bootstrap #324 ROC area: 0.522\n",
      "Bootstrap #325 ROC area: 0.693\n",
      "Bootstrap #326 ROC area: 0.668\n",
      "Bootstrap #327 ROC area: 0.618\n",
      "Bootstrap #328 ROC area: 0.710\n",
      "Bootstrap #329 ROC area: 0.717\n",
      "Bootstrap #330 ROC area: 0.514\n",
      "Bootstrap #331 ROC area: 0.617\n",
      "Bootstrap #332 ROC area: 0.655\n",
      "Bootstrap #333 ROC area: 0.630\n",
      "Bootstrap #334 ROC area: 0.613\n",
      "Bootstrap #335 ROC area: 0.743\n",
      "Bootstrap #336 ROC area: 0.751\n",
      "Bootstrap #337 ROC area: 0.629\n",
      "Bootstrap #338 ROC area: 0.569\n",
      "Bootstrap #339 ROC area: 0.721\n",
      "Bootstrap #340 ROC area: 0.694\n",
      "Bootstrap #341 ROC area: 0.648\n",
      "Bootstrap #342 ROC area: 0.674\n",
      "Bootstrap #343 ROC area: 0.619\n",
      "Bootstrap #344 ROC area: 0.685\n",
      "Bootstrap #345 ROC area: 0.728\n",
      "Bootstrap #346 ROC area: 0.679\n",
      "Bootstrap #347 ROC area: 0.680\n",
      "Bootstrap #348 ROC area: 0.590\n",
      "Bootstrap #349 ROC area: 0.748\n",
      "Bootstrap #350 ROC area: 0.754\n",
      "Bootstrap #351 ROC area: 0.693\n",
      "Bootstrap #352 ROC area: 0.646\n",
      "Bootstrap #353 ROC area: 0.612\n",
      "Bootstrap #354 ROC area: 0.739\n",
      "Bootstrap #355 ROC area: 0.783\n",
      "Bootstrap #356 ROC area: 0.565\n",
      "Bootstrap #357 ROC area: 0.635\n",
      "Bootstrap #358 ROC area: 0.588\n",
      "Bootstrap #359 ROC area: 0.691\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrap #360 ROC area: 0.486\n",
      "Bootstrap #361 ROC area: 0.578\n",
      "Bootstrap #362 ROC area: 0.634\n",
      "Bootstrap #363 ROC area: 0.710\n",
      "Bootstrap #364 ROC area: 0.726\n",
      "Bootstrap #365 ROC area: 0.762\n",
      "Bootstrap #366 ROC area: 0.591\n",
      "Bootstrap #367 ROC area: 0.665\n",
      "Bootstrap #368 ROC area: 0.634\n",
      "Bootstrap #369 ROC area: 0.710\n",
      "Bootstrap #370 ROC area: 0.691\n",
      "Bootstrap #371 ROC area: 0.610\n",
      "Bootstrap #372 ROC area: 0.672\n",
      "Bootstrap #373 ROC area: 0.757\n",
      "Bootstrap #374 ROC area: 0.679\n",
      "Bootstrap #375 ROC area: 0.776\n",
      "Bootstrap #376 ROC area: 0.663\n",
      "Bootstrap #377 ROC area: 0.642\n",
      "Bootstrap #378 ROC area: 0.709\n",
      "Bootstrap #379 ROC area: 0.603\n",
      "Bootstrap #380 ROC area: 0.694\n",
      "Bootstrap #381 ROC area: 0.620\n",
      "Bootstrap #382 ROC area: 0.661\n",
      "Bootstrap #383 ROC area: 0.762\n",
      "Bootstrap #384 ROC area: 0.630\n",
      "Bootstrap #385 ROC area: 0.597\n",
      "Bootstrap #386 ROC area: 0.745\n",
      "Bootstrap #387 ROC area: 0.606\n",
      "Bootstrap #388 ROC area: 0.730\n",
      "Bootstrap #389 ROC area: 0.710\n",
      "Bootstrap #390 ROC area: 0.699\n",
      "Bootstrap #391 ROC area: 0.663\n",
      "Bootstrap #392 ROC area: 0.671\n",
      "Bootstrap #393 ROC area: 0.769\n",
      "Bootstrap #394 ROC area: 0.570\n",
      "Bootstrap #395 ROC area: 0.601\n",
      "Bootstrap #396 ROC area: 0.654\n",
      "Bootstrap #397 ROC area: 0.681\n",
      "Bootstrap #398 ROC area: 0.727\n",
      "Bootstrap #399 ROC area: 0.630\n",
      "Bootstrap #400 ROC area: 0.654\n",
      "Bootstrap #401 ROC area: 0.706\n",
      "Bootstrap #402 ROC area: 0.797\n",
      "Bootstrap #403 ROC area: 0.676\n",
      "Bootstrap #404 ROC area: 0.671\n",
      "Bootstrap #405 ROC area: 0.711\n",
      "Bootstrap #406 ROC area: 0.717\n",
      "Bootstrap #407 ROC area: 0.703\n",
      "Bootstrap #408 ROC area: 0.771\n",
      "Bootstrap #409 ROC area: 0.677\n",
      "Bootstrap #410 ROC area: 0.699\n",
      "Bootstrap #411 ROC area: 0.654\n",
      "Bootstrap #412 ROC area: 0.580\n",
      "Bootstrap #413 ROC area: 0.703\n",
      "Bootstrap #414 ROC area: 0.754\n",
      "Bootstrap #415 ROC area: 0.735\n",
      "Bootstrap #416 ROC area: 0.630\n",
      "Bootstrap #417 ROC area: 0.587\n",
      "Bootstrap #418 ROC area: 0.699\n",
      "Bootstrap #419 ROC area: 0.856\n",
      "Bootstrap #420 ROC area: 0.753\n",
      "Bootstrap #421 ROC area: 0.707\n",
      "Bootstrap #422 ROC area: 0.708\n",
      "Bootstrap #423 ROC area: 0.700\n",
      "Bootstrap #424 ROC area: 0.685\n",
      "Bootstrap #425 ROC area: 0.705\n",
      "Bootstrap #426 ROC area: 0.643\n",
      "Bootstrap #427 ROC area: 0.778\n",
      "Bootstrap #428 ROC area: 0.625\n",
      "Bootstrap #429 ROC area: 0.604\n",
      "Bootstrap #430 ROC area: 0.654\n",
      "Bootstrap #431 ROC area: 0.569\n",
      "Bootstrap #432 ROC area: 0.637\n",
      "Bootstrap #433 ROC area: 0.671\n",
      "Bootstrap #434 ROC area: 0.784\n",
      "Bootstrap #435 ROC area: 0.710\n",
      "Bootstrap #436 ROC area: 0.656\n",
      "Bootstrap #437 ROC area: 0.731\n",
      "Bootstrap #438 ROC area: 0.714\n",
      "Bootstrap #439 ROC area: 0.725\n",
      "Bootstrap #440 ROC area: 0.643\n",
      "Bootstrap #441 ROC area: 0.678\n",
      "Bootstrap #442 ROC area: 0.826\n",
      "Bootstrap #443 ROC area: 0.596\n",
      "Bootstrap #444 ROC area: 0.608\n",
      "Bootstrap #445 ROC area: 0.806\n",
      "Bootstrap #446 ROC area: 0.592\n",
      "Bootstrap #447 ROC area: 0.803\n",
      "Bootstrap #448 ROC area: 0.689\n",
      "Bootstrap #449 ROC area: 0.796\n",
      "Bootstrap #450 ROC area: 0.649\n",
      "Bootstrap #451 ROC area: 0.629\n",
      "Bootstrap #452 ROC area: 0.801\n",
      "Bootstrap #453 ROC area: 0.701\n",
      "Bootstrap #454 ROC area: 0.696\n",
      "Bootstrap #455 ROC area: 0.610\n",
      "Bootstrap #456 ROC area: 0.751\n",
      "Bootstrap #457 ROC area: 0.642\n",
      "Bootstrap #458 ROC area: 0.783\n",
      "Bootstrap #459 ROC area: 0.660\n",
      "Bootstrap #460 ROC area: 0.653\n",
      "Bootstrap #461 ROC area: 0.741\n",
      "Bootstrap #462 ROC area: 0.674\n",
      "Bootstrap #463 ROC area: 0.815\n",
      "Bootstrap #464 ROC area: 0.693\n",
      "Bootstrap #465 ROC area: 0.594\n",
      "Bootstrap #466 ROC area: 0.771\n",
      "Bootstrap #467 ROC area: 0.612\n",
      "Bootstrap #468 ROC area: 0.612\n",
      "Bootstrap #469 ROC area: 0.697\n",
      "Bootstrap #470 ROC area: 0.780\n",
      "Bootstrap #471 ROC area: 0.618\n",
      "Bootstrap #472 ROC area: 0.821\n",
      "Bootstrap #473 ROC area: 0.651\n",
      "Bootstrap #474 ROC area: 0.457\n",
      "Bootstrap #475 ROC area: 0.619\n",
      "Bootstrap #476 ROC area: 0.548\n",
      "Bootstrap #477 ROC area: 0.634\n",
      "Bootstrap #478 ROC area: 0.755\n",
      "Bootstrap #479 ROC area: 0.680\n",
      "Bootstrap #480 ROC area: 0.752\n",
      "Bootstrap #481 ROC area: 0.755\n",
      "Bootstrap #482 ROC area: 0.604\n",
      "Bootstrap #483 ROC area: 0.555\n",
      "Bootstrap #484 ROC area: 0.638\n",
      "Bootstrap #485 ROC area: 0.698\n",
      "Bootstrap #486 ROC area: 0.732\n",
      "Bootstrap #487 ROC area: 0.522\n",
      "Bootstrap #488 ROC area: 0.763\n",
      "Bootstrap #489 ROC area: 0.760\n",
      "Bootstrap #490 ROC area: 0.699\n",
      "Bootstrap #491 ROC area: 0.719\n",
      "Bootstrap #492 ROC area: 0.739\n",
      "Bootstrap #493 ROC area: 0.600\n",
      "Bootstrap #494 ROC area: 0.575\n",
      "Bootstrap #495 ROC area: 0.640\n",
      "Bootstrap #496 ROC area: 0.590\n",
      "Bootstrap #497 ROC area: 0.612\n",
      "Bootstrap #498 ROC area: 0.641\n",
      "Bootstrap #499 ROC area: 0.750\n",
      "Bootstrap #500 ROC area: 0.645\n",
      "Bootstrap #501 ROC area: 0.679\n",
      "Bootstrap #502 ROC area: 0.636\n",
      "Bootstrap #503 ROC area: 0.689\n",
      "Bootstrap #504 ROC area: 0.680\n",
      "Bootstrap #505 ROC area: 0.617\n",
      "Bootstrap #506 ROC area: 0.607\n",
      "Bootstrap #507 ROC area: 0.556\n",
      "Bootstrap #508 ROC area: 0.724\n",
      "Bootstrap #509 ROC area: 0.640\n",
      "Bootstrap #510 ROC area: 0.588\n",
      "Bootstrap #511 ROC area: 0.621\n",
      "Bootstrap #512 ROC area: 0.776\n",
      "Bootstrap #513 ROC area: 0.682\n",
      "Bootstrap #514 ROC area: 0.563\n",
      "Bootstrap #515 ROC area: 0.692\n",
      "Bootstrap #516 ROC area: 0.598\n",
      "Bootstrap #517 ROC area: 0.695\n",
      "Bootstrap #518 ROC area: 0.693\n",
      "Bootstrap #519 ROC area: 0.733\n",
      "Bootstrap #520 ROC area: 0.725\n",
      "Bootstrap #521 ROC area: 0.634\n",
      "Bootstrap #522 ROC area: 0.740\n",
      "Bootstrap #523 ROC area: 0.567\n",
      "Bootstrap #524 ROC area: 0.683\n",
      "Bootstrap #525 ROC area: 0.601\n",
      "Bootstrap #526 ROC area: 0.587\n",
      "Bootstrap #527 ROC area: 0.701\n",
      "Bootstrap #528 ROC area: 0.637\n",
      "Bootstrap #529 ROC area: 0.643\n",
      "Bootstrap #530 ROC area: 0.644\n",
      "Bootstrap #531 ROC area: 0.716\n",
      "Bootstrap #532 ROC area: 0.692\n",
      "Bootstrap #533 ROC area: 0.666\n",
      "Bootstrap #534 ROC area: 0.655\n",
      "Bootstrap #535 ROC area: 0.627\n",
      "Bootstrap #536 ROC area: 0.653\n",
      "Bootstrap #537 ROC area: 0.623\n",
      "Bootstrap #538 ROC area: 0.595\n",
      "Bootstrap #539 ROC area: 0.588\n",
      "Bootstrap #540 ROC area: 0.826\n",
      "Bootstrap #541 ROC area: 0.619\n",
      "Bootstrap #542 ROC area: 0.631\n",
      "Bootstrap #543 ROC area: 0.588\n",
      "Bootstrap #544 ROC area: 0.762\n",
      "Bootstrap #545 ROC area: 0.673\n",
      "Bootstrap #546 ROC area: 0.725\n",
      "Bootstrap #547 ROC area: 0.631\n",
      "Bootstrap #548 ROC area: 0.704\n",
      "Bootstrap #549 ROC area: 0.697\n",
      "Bootstrap #550 ROC area: 0.750\n",
      "Bootstrap #551 ROC area: 0.555\n",
      "Bootstrap #552 ROC area: 0.692\n",
      "Bootstrap #553 ROC area: 0.681\n",
      "Bootstrap #554 ROC area: 0.549\n",
      "Bootstrap #555 ROC area: 0.643\n",
      "Bootstrap #556 ROC area: 0.733\n",
      "Bootstrap #557 ROC area: 0.719\n",
      "Bootstrap #558 ROC area: 0.596\n",
      "Bootstrap #559 ROC area: 0.658\n",
      "Bootstrap #560 ROC area: 0.680\n",
      "Bootstrap #561 ROC area: 0.669\n",
      "Bootstrap #562 ROC area: 0.637\n",
      "Bootstrap #563 ROC area: 0.682\n",
      "Bootstrap #564 ROC area: 0.819\n",
      "Bootstrap #565 ROC area: 0.636\n",
      "Bootstrap #566 ROC area: 0.677\n",
      "Bootstrap #567 ROC area: 0.755\n",
      "Bootstrap #568 ROC area: 0.654\n",
      "Bootstrap #569 ROC area: 0.671\n",
      "Bootstrap #570 ROC area: 0.764\n",
      "Bootstrap #571 ROC area: 0.557\n",
      "Bootstrap #572 ROC area: 0.648\n",
      "Bootstrap #573 ROC area: 0.716\n",
      "Bootstrap #574 ROC area: 0.622\n",
      "Bootstrap #575 ROC area: 0.702\n",
      "Bootstrap #576 ROC area: 0.608\n",
      "Bootstrap #577 ROC area: 0.635\n",
      "Bootstrap #578 ROC area: 0.705\n",
      "Bootstrap #579 ROC area: 0.712\n",
      "Bootstrap #580 ROC area: 0.667\n",
      "Bootstrap #581 ROC area: 0.715\n",
      "Bootstrap #582 ROC area: 0.510\n",
      "Bootstrap #583 ROC area: 0.625\n",
      "Bootstrap #584 ROC area: 0.685\n",
      "Bootstrap #585 ROC area: 0.665\n",
      "Bootstrap #586 ROC area: 0.560\n",
      "Bootstrap #587 ROC area: 0.723\n",
      "Bootstrap #588 ROC area: 0.569\n",
      "Bootstrap #589 ROC area: 0.750\n",
      "Bootstrap #590 ROC area: 0.658\n",
      "Bootstrap #591 ROC area: 0.585\n",
      "Bootstrap #592 ROC area: 0.676\n",
      "Bootstrap #593 ROC area: 0.725\n",
      "Bootstrap #594 ROC area: 0.713\n",
      "Bootstrap #595 ROC area: 0.716\n",
      "Bootstrap #596 ROC area: 0.701\n",
      "Bootstrap #597 ROC area: 0.642\n",
      "Bootstrap #598 ROC area: 0.673\n",
      "Bootstrap #599 ROC area: 0.656\n",
      "Bootstrap #600 ROC area: 0.754\n",
      "Bootstrap #601 ROC area: 0.755\n",
      "Bootstrap #602 ROC area: 0.629\n",
      "Bootstrap #603 ROC area: 0.720\n",
      "Bootstrap #604 ROC area: 0.556\n",
      "Bootstrap #605 ROC area: 0.621\n",
      "Bootstrap #606 ROC area: 0.660\n",
      "Bootstrap #607 ROC area: 0.824\n",
      "Bootstrap #608 ROC area: 0.756\n",
      "Bootstrap #609 ROC area: 0.702\n",
      "Bootstrap #610 ROC area: 0.619\n",
      "Bootstrap #611 ROC area: 0.642\n",
      "Bootstrap #612 ROC area: 0.809\n",
      "Bootstrap #613 ROC area: 0.720\n",
      "Bootstrap #614 ROC area: 0.731\n",
      "Bootstrap #615 ROC area: 0.577\n",
      "Bootstrap #616 ROC area: 0.533\n",
      "Bootstrap #617 ROC area: 0.613\n",
      "Bootstrap #618 ROC area: 0.763\n",
      "Bootstrap #619 ROC area: 0.582\n",
      "Bootstrap #620 ROC area: 0.654\n",
      "Bootstrap #621 ROC area: 0.638\n",
      "Bootstrap #622 ROC area: 0.726\n",
      "Bootstrap #623 ROC area: 0.615\n",
      "Bootstrap #624 ROC area: 0.718\n",
      "Bootstrap #625 ROC area: 0.698\n",
      "Bootstrap #626 ROC area: 0.690\n",
      "Bootstrap #627 ROC area: 0.725\n",
      "Bootstrap #628 ROC area: 0.660\n",
      "Bootstrap #629 ROC area: 0.628\n",
      "Bootstrap #630 ROC area: 0.603\n",
      "Bootstrap #631 ROC area: 0.588\n",
      "Bootstrap #632 ROC area: 0.670\n",
      "Bootstrap #633 ROC area: 0.660\n",
      "Bootstrap #634 ROC area: 0.665\n",
      "Bootstrap #635 ROC area: 0.737\n",
      "Bootstrap #636 ROC area: 0.625\n",
      "Bootstrap #637 ROC area: 0.708\n",
      "Bootstrap #638 ROC area: 0.635\n",
      "Bootstrap #639 ROC area: 0.661\n",
      "Bootstrap #640 ROC area: 0.540\n",
      "Bootstrap #641 ROC area: 0.765\n",
      "Bootstrap #642 ROC area: 0.598\n",
      "Bootstrap #643 ROC area: 0.730\n",
      "Bootstrap #644 ROC area: 0.632\n",
      "Bootstrap #645 ROC area: 0.704\n",
      "Bootstrap #646 ROC area: 0.602\n",
      "Bootstrap #647 ROC area: 0.630\n",
      "Bootstrap #648 ROC area: 0.745\n",
      "Bootstrap #649 ROC area: 0.640\n",
      "Bootstrap #650 ROC area: 0.763\n",
      "Bootstrap #651 ROC area: 0.678\n",
      "Bootstrap #652 ROC area: 0.646\n",
      "Bootstrap #653 ROC area: 0.663\n",
      "Bootstrap #654 ROC area: 0.715\n",
      "Bootstrap #655 ROC area: 0.633\n",
      "Bootstrap #656 ROC area: 0.577\n",
      "Bootstrap #657 ROC area: 0.574\n",
      "Bootstrap #658 ROC area: 0.621\n",
      "Bootstrap #659 ROC area: 0.754\n",
      "Bootstrap #660 ROC area: 0.657\n",
      "Bootstrap #661 ROC area: 0.698\n",
      "Bootstrap #662 ROC area: 0.621\n",
      "Bootstrap #663 ROC area: 0.604\n",
      "Bootstrap #664 ROC area: 0.606\n",
      "Bootstrap #665 ROC area: 0.628\n",
      "Bootstrap #666 ROC area: 0.768\n",
      "Bootstrap #667 ROC area: 0.582\n",
      "Bootstrap #668 ROC area: 0.750\n",
      "Bootstrap #669 ROC area: 0.614\n",
      "Bootstrap #670 ROC area: 0.598\n",
      "Bootstrap #671 ROC area: 0.727\n",
      "Bootstrap #672 ROC area: 0.606\n",
      "Bootstrap #673 ROC area: 0.682\n",
      "Bootstrap #674 ROC area: 0.641\n",
      "Bootstrap #675 ROC area: 0.574\n",
      "Bootstrap #676 ROC area: 0.745\n",
      "Bootstrap #677 ROC area: 0.697\n",
      "Bootstrap #678 ROC area: 0.715\n",
      "Bootstrap #679 ROC area: 0.722\n",
      "Bootstrap #680 ROC area: 0.667\n",
      "Bootstrap #681 ROC area: 0.777\n",
      "Bootstrap #682 ROC area: 0.651\n",
      "Bootstrap #683 ROC area: 0.771\n",
      "Bootstrap #684 ROC area: 0.626\n",
      "Bootstrap #685 ROC area: 0.665\n",
      "Bootstrap #686 ROC area: 0.573\n",
      "Bootstrap #687 ROC area: 0.655\n",
      "Bootstrap #688 ROC area: 0.510\n",
      "Bootstrap #689 ROC area: 0.815\n",
      "Bootstrap #690 ROC area: 0.609\n",
      "Bootstrap #691 ROC area: 0.728\n",
      "Bootstrap #692 ROC area: 0.798\n",
      "Bootstrap #693 ROC area: 0.668\n",
      "Bootstrap #694 ROC area: 0.800\n",
      "Bootstrap #695 ROC area: 0.675\n",
      "Bootstrap #696 ROC area: 0.733\n",
      "Bootstrap #697 ROC area: 0.703\n",
      "Bootstrap #698 ROC area: 0.523\n",
      "Bootstrap #699 ROC area: 0.653\n",
      "Bootstrap #700 ROC area: 0.722\n",
      "Bootstrap #701 ROC area: 0.725\n",
      "Bootstrap #702 ROC area: 0.641\n",
      "Bootstrap #703 ROC area: 0.648\n",
      "Bootstrap #704 ROC area: 0.705\n",
      "Bootstrap #705 ROC area: 0.689\n",
      "Bootstrap #706 ROC area: 0.572\n",
      "Bootstrap #707 ROC area: 0.687\n",
      "Bootstrap #708 ROC area: 0.827\n",
      "Bootstrap #709 ROC area: 0.624\n",
      "Bootstrap #710 ROC area: 0.671\n",
      "Bootstrap #711 ROC area: 0.680\n",
      "Bootstrap #712 ROC area: 0.649\n",
      "Bootstrap #713 ROC area: 0.738\n",
      "Bootstrap #714 ROC area: 0.634\n",
      "Bootstrap #715 ROC area: 0.669\n",
      "Bootstrap #716 ROC area: 0.664\n",
      "Bootstrap #717 ROC area: 0.548\n",
      "Bootstrap #718 ROC area: 0.570\n",
      "Bootstrap #719 ROC area: 0.544\n",
      "Bootstrap #720 ROC area: 0.651\n",
      "Bootstrap #721 ROC area: 0.710\n",
      "Bootstrap #722 ROC area: 0.661\n",
      "Bootstrap #723 ROC area: 0.664\n",
      "Bootstrap #724 ROC area: 0.664\n",
      "Bootstrap #725 ROC area: 0.660\n",
      "Bootstrap #726 ROC area: 0.647\n",
      "Bootstrap #727 ROC area: 0.685\n",
      "Bootstrap #728 ROC area: 0.623\n",
      "Bootstrap #729 ROC area: 0.636\n",
      "Bootstrap #730 ROC area: 0.696\n",
      "Bootstrap #731 ROC area: 0.638\n",
      "Bootstrap #732 ROC area: 0.798\n",
      "Bootstrap #733 ROC area: 0.596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrap #734 ROC area: 0.701\n",
      "Bootstrap #735 ROC area: 0.722\n",
      "Bootstrap #736 ROC area: 0.624\n",
      "Bootstrap #737 ROC area: 0.567\n",
      "Bootstrap #738 ROC area: 0.712\n",
      "Bootstrap #739 ROC area: 0.745\n",
      "Bootstrap #740 ROC area: 0.712\n",
      "Bootstrap #741 ROC area: 0.689\n",
      "Bootstrap #742 ROC area: 0.687\n",
      "Bootstrap #743 ROC area: 0.657\n",
      "Bootstrap #744 ROC area: 0.617\n",
      "Bootstrap #745 ROC area: 0.650\n",
      "Bootstrap #746 ROC area: 0.678\n",
      "Bootstrap #747 ROC area: 0.588\n",
      "Bootstrap #748 ROC area: 0.600\n",
      "Bootstrap #749 ROC area: 0.584\n",
      "Bootstrap #750 ROC area: 0.668\n",
      "Bootstrap #751 ROC area: 0.578\n",
      "Bootstrap #752 ROC area: 0.701\n",
      "Bootstrap #753 ROC area: 0.550\n",
      "Bootstrap #754 ROC area: 0.522\n",
      "Bootstrap #755 ROC area: 0.618\n",
      "Bootstrap #756 ROC area: 0.721\n",
      "Bootstrap #757 ROC area: 0.678\n",
      "Bootstrap #758 ROC area: 0.697\n",
      "Bootstrap #759 ROC area: 0.614\n",
      "Bootstrap #760 ROC area: 0.621\n",
      "Bootstrap #761 ROC area: 0.662\n",
      "Bootstrap #762 ROC area: 0.680\n",
      "Bootstrap #763 ROC area: 0.648\n",
      "Bootstrap #764 ROC area: 0.677\n",
      "Bootstrap #765 ROC area: 0.691\n",
      "Bootstrap #766 ROC area: 0.512\n",
      "Bootstrap #767 ROC area: 0.791\n",
      "Bootstrap #768 ROC area: 0.521\n",
      "Bootstrap #769 ROC area: 0.646\n",
      "Bootstrap #770 ROC area: 0.693\n",
      "Bootstrap #771 ROC area: 0.625\n",
      "Bootstrap #772 ROC area: 0.614\n",
      "Bootstrap #773 ROC area: 0.461\n",
      "Bootstrap #774 ROC area: 0.685\n",
      "Bootstrap #775 ROC area: 0.713\n",
      "Bootstrap #776 ROC area: 0.617\n",
      "Bootstrap #777 ROC area: 0.624\n",
      "Bootstrap #778 ROC area: 0.579\n",
      "Bootstrap #779 ROC area: 0.656\n",
      "Bootstrap #780 ROC area: 0.730\n",
      "Bootstrap #781 ROC area: 0.542\n",
      "Bootstrap #782 ROC area: 0.679\n",
      "Bootstrap #783 ROC area: 0.570\n",
      "Bootstrap #784 ROC area: 0.687\n",
      "Bootstrap #785 ROC area: 0.677\n",
      "Bootstrap #786 ROC area: 0.573\n",
      "Bootstrap #787 ROC area: 0.550\n",
      "Bootstrap #788 ROC area: 0.688\n",
      "Bootstrap #789 ROC area: 0.595\n",
      "Bootstrap #790 ROC area: 0.690\n",
      "Bootstrap #791 ROC area: 0.657\n",
      "Bootstrap #792 ROC area: 0.609\n",
      "Bootstrap #793 ROC area: 0.704\n",
      "Bootstrap #794 ROC area: 0.681\n",
      "Bootstrap #795 ROC area: 0.701\n",
      "Bootstrap #796 ROC area: 0.614\n",
      "Bootstrap #797 ROC area: 0.582\n",
      "Bootstrap #798 ROC area: 0.573\n",
      "Bootstrap #799 ROC area: 0.650\n",
      "Bootstrap #800 ROC area: 0.700\n",
      "Bootstrap #801 ROC area: 0.705\n",
      "Bootstrap #802 ROC area: 0.713\n",
      "Bootstrap #803 ROC area: 0.585\n",
      "Bootstrap #804 ROC area: 0.621\n",
      "Bootstrap #805 ROC area: 0.600\n",
      "Bootstrap #806 ROC area: 0.793\n",
      "Bootstrap #807 ROC area: 0.660\n",
      "Bootstrap #808 ROC area: 0.739\n",
      "Bootstrap #809 ROC area: 0.784\n",
      "Bootstrap #810 ROC area: 0.613\n",
      "Bootstrap #811 ROC area: 0.682\n",
      "Bootstrap #812 ROC area: 0.647\n",
      "Bootstrap #813 ROC area: 0.721\n",
      "Bootstrap #814 ROC area: 0.595\n",
      "Bootstrap #815 ROC area: 0.685\n",
      "Bootstrap #816 ROC area: 0.663\n",
      "Bootstrap #817 ROC area: 0.668\n",
      "Bootstrap #818 ROC area: 0.697\n",
      "Bootstrap #819 ROC area: 0.538\n",
      "Bootstrap #820 ROC area: 0.623\n",
      "Bootstrap #821 ROC area: 0.684\n",
      "Bootstrap #822 ROC area: 0.643\n",
      "Bootstrap #823 ROC area: 0.616\n",
      "Bootstrap #824 ROC area: 0.691\n",
      "Bootstrap #825 ROC area: 0.795\n",
      "Bootstrap #826 ROC area: 0.747\n",
      "Bootstrap #827 ROC area: 0.639\n",
      "Bootstrap #828 ROC area: 0.540\n",
      "Bootstrap #829 ROC area: 0.628\n",
      "Bootstrap #830 ROC area: 0.555\n",
      "Bootstrap #831 ROC area: 0.667\n",
      "Bootstrap #832 ROC area: 0.661\n",
      "Bootstrap #833 ROC area: 0.646\n",
      "Bootstrap #834 ROC area: 0.681\n",
      "Bootstrap #835 ROC area: 0.612\n",
      "Bootstrap #836 ROC area: 0.556\n",
      "Bootstrap #837 ROC area: 0.734\n",
      "Bootstrap #838 ROC area: 0.603\n",
      "Bootstrap #839 ROC area: 0.747\n",
      "Bootstrap #840 ROC area: 0.544\n",
      "Bootstrap #841 ROC area: 0.621\n",
      "Bootstrap #842 ROC area: 0.598\n",
      "Bootstrap #843 ROC area: 0.588\n",
      "Bootstrap #844 ROC area: 0.598\n",
      "Bootstrap #845 ROC area: 0.523\n",
      "Bootstrap #846 ROC area: 0.619\n",
      "Bootstrap #847 ROC area: 0.715\n",
      "Bootstrap #848 ROC area: 0.705\n",
      "Bootstrap #849 ROC area: 0.601\n",
      "Bootstrap #850 ROC area: 0.688\n",
      "Bootstrap #851 ROC area: 0.729\n",
      "Bootstrap #852 ROC area: 0.711\n",
      "Bootstrap #853 ROC area: 0.723\n",
      "Bootstrap #854 ROC area: 0.750\n",
      "Bootstrap #855 ROC area: 0.677\n",
      "Bootstrap #856 ROC area: 0.651\n",
      "Bootstrap #857 ROC area: 0.663\n",
      "Bootstrap #858 ROC area: 0.556\n",
      "Bootstrap #859 ROC area: 0.667\n",
      "Bootstrap #860 ROC area: 0.727\n",
      "Bootstrap #861 ROC area: 0.749\n",
      "Bootstrap #862 ROC area: 0.677\n",
      "Bootstrap #863 ROC area: 0.624\n",
      "Bootstrap #864 ROC area: 0.780\n",
      "Bootstrap #865 ROC area: 0.579\n",
      "Bootstrap #866 ROC area: 0.735\n",
      "Bootstrap #867 ROC area: 0.713\n",
      "Bootstrap #868 ROC area: 0.755\n",
      "Bootstrap #869 ROC area: 0.722\n",
      "Bootstrap #870 ROC area: 0.696\n",
      "Bootstrap #871 ROC area: 0.715\n",
      "Bootstrap #872 ROC area: 0.660\n",
      "Bootstrap #873 ROC area: 0.747\n",
      "Bootstrap #874 ROC area: 0.471\n",
      "Bootstrap #875 ROC area: 0.616\n",
      "Bootstrap #876 ROC area: 0.668\n",
      "Bootstrap #877 ROC area: 0.783\n",
      "Bootstrap #878 ROC area: 0.496\n",
      "Bootstrap #879 ROC area: 0.628\n",
      "Bootstrap #880 ROC area: 0.630\n",
      "Bootstrap #881 ROC area: 0.651\n",
      "Bootstrap #882 ROC area: 0.634\n",
      "Bootstrap #883 ROC area: 0.567\n",
      "Bootstrap #884 ROC area: 0.663\n",
      "Bootstrap #885 ROC area: 0.717\n",
      "Bootstrap #886 ROC area: 0.653\n",
      "Bootstrap #887 ROC area: 0.655\n",
      "Bootstrap #888 ROC area: 0.683\n",
      "Bootstrap #889 ROC area: 0.690\n",
      "Bootstrap #890 ROC area: 0.629\n",
      "Bootstrap #891 ROC area: 0.816\n",
      "Bootstrap #892 ROC area: 0.734\n",
      "Bootstrap #893 ROC area: 0.625\n",
      "Bootstrap #894 ROC area: 0.752\n",
      "Bootstrap #895 ROC area: 0.662\n",
      "Bootstrap #896 ROC area: 0.675\n",
      "Bootstrap #897 ROC area: 0.727\n",
      "Bootstrap #898 ROC area: 0.616\n",
      "Bootstrap #899 ROC area: 0.788\n",
      "Bootstrap #900 ROC area: 0.777\n",
      "Bootstrap #901 ROC area: 0.804\n",
      "Bootstrap #902 ROC area: 0.581\n",
      "Bootstrap #903 ROC area: 0.602\n",
      "Bootstrap #904 ROC area: 0.677\n",
      "Bootstrap #905 ROC area: 0.773\n",
      "Bootstrap #906 ROC area: 0.594\n",
      "Bootstrap #907 ROC area: 0.687\n",
      "Bootstrap #908 ROC area: 0.592\n",
      "Bootstrap #909 ROC area: 0.692\n",
      "Bootstrap #910 ROC area: 0.660\n",
      "Bootstrap #911 ROC area: 0.623\n",
      "Bootstrap #912 ROC area: 0.669\n",
      "Bootstrap #913 ROC area: 0.674\n",
      "Bootstrap #914 ROC area: 0.676\n",
      "Bootstrap #915 ROC area: 0.599\n",
      "Bootstrap #916 ROC area: 0.573\n",
      "Bootstrap #917 ROC area: 0.579\n",
      "Bootstrap #918 ROC area: 0.757\n",
      "Bootstrap #919 ROC area: 0.639\n",
      "Bootstrap #920 ROC area: 0.712\n",
      "Bootstrap #921 ROC area: 0.630\n",
      "Bootstrap #922 ROC area: 0.723\n",
      "Bootstrap #923 ROC area: 0.592\n",
      "Bootstrap #924 ROC area: 0.680\n",
      "Bootstrap #925 ROC area: 0.629\n",
      "Bootstrap #926 ROC area: 0.564\n",
      "Bootstrap #927 ROC area: 0.645\n",
      "Bootstrap #928 ROC area: 0.684\n",
      "Bootstrap #929 ROC area: 0.571\n",
      "Bootstrap #930 ROC area: 0.720\n",
      "Bootstrap #931 ROC area: 0.661\n",
      "Bootstrap #932 ROC area: 0.623\n",
      "Bootstrap #933 ROC area: 0.688\n",
      "Bootstrap #934 ROC area: 0.729\n",
      "Bootstrap #935 ROC area: 0.725\n",
      "Bootstrap #936 ROC area: 0.781\n",
      "Bootstrap #937 ROC area: 0.615\n",
      "Bootstrap #938 ROC area: 0.718\n",
      "Bootstrap #939 ROC area: 0.824\n",
      "Bootstrap #940 ROC area: 0.607\n",
      "Bootstrap #941 ROC area: 0.623\n",
      "Bootstrap #942 ROC area: 0.668\n",
      "Bootstrap #943 ROC area: 0.711\n",
      "Bootstrap #944 ROC area: 0.697\n",
      "Bootstrap #945 ROC area: 0.779\n",
      "Bootstrap #946 ROC area: 0.697\n",
      "Bootstrap #947 ROC area: 0.714\n",
      "Bootstrap #948 ROC area: 0.779\n",
      "Bootstrap #949 ROC area: 0.624\n",
      "Bootstrap #950 ROC area: 0.632\n",
      "Bootstrap #951 ROC area: 0.677\n",
      "Bootstrap #952 ROC area: 0.600\n",
      "Bootstrap #953 ROC area: 0.587\n",
      "Bootstrap #954 ROC area: 0.736\n",
      "Bootstrap #955 ROC area: 0.563\n",
      "Bootstrap #956 ROC area: 0.775\n",
      "Bootstrap #957 ROC area: 0.785\n",
      "Bootstrap #958 ROC area: 0.656\n",
      "Bootstrap #959 ROC area: 0.670\n",
      "Bootstrap #960 ROC area: 0.715\n",
      "Bootstrap #961 ROC area: 0.716\n",
      "Bootstrap #962 ROC area: 0.772\n",
      "Bootstrap #963 ROC area: 0.607\n",
      "Bootstrap #964 ROC area: 0.698\n",
      "Bootstrap #965 ROC area: 0.529\n",
      "Bootstrap #966 ROC area: 0.688\n",
      "Bootstrap #967 ROC area: 0.751\n",
      "Bootstrap #968 ROC area: 0.754\n",
      "Bootstrap #969 ROC area: 0.630\n",
      "Bootstrap #970 ROC area: 0.556\n",
      "Bootstrap #971 ROC area: 0.517\n",
      "Bootstrap #972 ROC area: 0.662\n",
      "Bootstrap #973 ROC area: 0.709\n",
      "Bootstrap #974 ROC area: 0.720\n",
      "Bootstrap #975 ROC area: 0.822\n",
      "Bootstrap #976 ROC area: 0.522\n",
      "Bootstrap #977 ROC area: 0.711\n",
      "Bootstrap #978 ROC area: 0.690\n",
      "Bootstrap #979 ROC area: 0.637\n",
      "Bootstrap #980 ROC area: 0.611\n",
      "Bootstrap #981 ROC area: 0.686\n",
      "Bootstrap #982 ROC area: 0.775\n",
      "Bootstrap #983 ROC area: 0.483\n",
      "Bootstrap #984 ROC area: 0.647\n",
      "Bootstrap #985 ROC area: 0.707\n",
      "Bootstrap #986 ROC area: 0.655\n",
      "Bootstrap #987 ROC area: 0.619\n",
      "Bootstrap #988 ROC area: 0.692\n",
      "Bootstrap #989 ROC area: 0.676\n",
      "Bootstrap #990 ROC area: 0.550\n",
      "Bootstrap #991 ROC area: 0.637\n",
      "Bootstrap #992 ROC area: 0.704\n",
      "Bootstrap #993 ROC area: 0.788\n",
      "Bootstrap #994 ROC area: 0.829\n",
      "Bootstrap #995 ROC area: 0.709\n",
      "Bootstrap #996 ROC area: 0.674\n",
      "Bootstrap #997 ROC area: 0.708\n",
      "Bootstrap #998 ROC area: 0.617\n",
      "Bootstrap #999 ROC area: 0.632\n",
      "Bootstrap #1000 ROC area: 0.849\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import sem\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "y_pred = y_prob1\n",
    "y_true = y_test\n",
    "\n",
    "print(\"Original ROC area: {:0.3f}\".format(roc_auc_score(y_true, y_pred)))\n",
    "\n",
    "n_bootstraps = 1000\n",
    "rng_seed = 42  # control reproducibility\n",
    "bootstrapped_scores = []\n",
    "\n",
    "rng = np.random.RandomState(rng_seed)\n",
    "for i in range(n_bootstraps):\n",
    "    # bootstrap by sampling with replacement on the prediction indices\n",
    "    indices = rng.randint(0, len(y_pred), len(y_pred))\n",
    "    if len(np.unique(y_true[indices])) < 2:\n",
    "        # We need at least one positive and one negative sample for ROC AUC\n",
    "        # to be defined: reject the sample\n",
    "        continue\n",
    "\n",
    "    score = roc_auc_score(y_true[indices], y_pred[indices])\n",
    "    bootstrapped_scores.append(score)\n",
    "    print(\"Bootstrap #{} ROC area: {:0.3f}\".format(i + 1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5122752b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC 95%CI:[0.550-0.78]\n"
     ]
    }
   ],
   "source": [
    "#计算AUC95% CI\n",
    "sorted_scores = np.array(bootstrapped_scores)\n",
    "sorted_scores.sort()\n",
    "\n",
    "# Computing the lower and upper bound of the 90% confidence interval\n",
    "# You can change the bounds percentiles to 0.025 and 0.975 to get\n",
    "# a 95% confidence interval instead.\n",
    "confidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]\n",
    "confidence_upper = sorted_scores[int(0.95 * len(sorted_scores))]\n",
    "print(\"AUC 95%CI:[{:0.3f}-{:0.3}]\".format(\n",
    "    confidence_lower, confidence_upper))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c21ad80d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 578692 characters]",
      "text/plain": [
       "<Figure size 4800x4800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc\n",
    "import matplotlib.pyplot as plt\n",
    "fpr, tpr, threshold = roc_curve(y_test,y_prob1)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "plt.figure(figsize=(4, 4), dpi=1200)\n",
    "plt.plot(fpr, tpr, color='darkorange', lw=1.5, label='AUC = %0.3f' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=1.5, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('SVM ROC Curve')\n",
    "plt.legend(['AUC: %0.3f' % roc_auc, \"95%CI: {:0.3f}-{:0.3}\".format(\n",
    "    confidence_lower, confidence_upper)],loc=\"lower right\", prop={'size': 8},markerfirst=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5e7d3532",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6680180180180181"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0bb9032a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 445396 characters]",
      "text/plain": [
       "<Figure size 4800x4800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import precision_recall_curve, average_precision_score\n",
    "\n",
    "# y_true为样本实际的类别1为正例0为反例，y_scores为阈值\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test,y_prob1)\n",
    "AP = average_precision_score(y_test,y_prob1, average='macro', pos_label=1, sample_weight=None)\n",
    "plt.figure(figsize=(4, 4), dpi=1200)\n",
    "plt.plot(recall, precision, lw=1.5, color='navy', label='PR curve (AP = %0.3f)' % AP)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.title('SVM P-R Curve')\n",
    "plt.legend(loc=(0.5,0.1), prop={'size': 7})#标签左下角坐标，字体7号\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5b93ae62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AP: 0.6951323935453104\n"
     ]
    }
   ],
   "source": [
    "#计算AP值，PR曲线下面积，PR曲线下面的面积越大，模型的性能越好\n",
    "from sklearn.metrics import precision_recall_curve, average_precision_score\n",
    "AP = average_precision_score(y_test,y_prob1, average='macro', pos_label=1, sample_weight=None)\n",
    "print('AP:', AP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6cca7123",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Brier评分: 0.23422182468325708\n"
     ]
    }
   ],
   "source": [
    "##brier评分\n",
    "from sklearn.metrics import brier_score_loss\n",
    "print('Brier评分:', brier_score_loss(y_test,y_prob1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfaaf261",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
