{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "plastic-dealer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "from glob import glob\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import random\n",
    "from random import sample\n",
    "import sklearn.model_selection as skl\n",
    "#from kaggle_datasets import KaggleDatasets\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from keras_preprocessing.image.dataframe_iterator import DataFrameIterator\n",
    "from tensorflow.keras.applications import EfficientNetB3\n",
    "from tensorflow.keras.layers import InputLayer, GlobalAveragePooling2D, BatchNormalization, Dense, Dropout, Flatten, Conv2D, MaxPooling2D \n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.metrics import AUC\n",
    "from tensorflow.keras.optimizers import RMSprop\n",
    "import tensorflow_hub as tfhub\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\n",
    "\n",
    "import pydicom\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "secure-password",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = 'E:/MyMasterthesis/Ref/Dataset/'\n",
    "df = pd.read_csv('E:/MyMasterthesis/Ref/Dataset/train_labels.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "logical-framework",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BraTS21ID</th>\n",
       "      <th>MGMT_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BraTS21ID  MGMT_value\n",
       "0          0           1\n",
       "1          2           1\n",
       "2          3           0\n",
       "3          5           1\n",
       "4          6           1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "stretch-bottle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the full paths for each id for different types of sequences to the csv \n",
    "def full_ids(data):\n",
    "    zeros = 5 - len(str(data))\n",
    "    if zeros > 0:\n",
    "        prefix = ''.join(['0' for i in range(zeros)])\n",
    "    \n",
    "    return prefix+str(data)\n",
    "        \n",
    "\n",
    "df['BraTS21ID_full'] = df['BraTS21ID'].apply(full_ids)\n",
    "\n",
    "# Add all the paths to the df for easy access\n",
    "df['flair'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/FLAIR/')\n",
    "df['t1w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1w/')\n",
    "df['t1wce'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1wCE/')\n",
    "df['t2w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T2w/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "practical-warren",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(root_dir+'sample_submission.csv')\n",
    "\n",
    "df_test['BraTS21ID_full'] = df_test['BraTS21ID'].apply(full_ids)\n",
    "\n",
    "# Add all the paths to the df for easy access\n",
    "df_test['flair'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/FLAIR/')\n",
    "df_test['t1w'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T1w/')\n",
    "df_test['t1wce'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T1wCE/')\n",
    "df_test['t2w'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T2w/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "monetary-applicant",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_val_dataframe(mri_type):\n",
    "    \n",
    "    all_img_files = []\n",
    "    all_img_labels = []\n",
    "    all_img_patient_ids = []\n",
    "    for row in df.iterrows():\n",
    "        if row[1]['BraTS21ID_full'] == '00109' and mri_type == 'flair':\n",
    "            continue\n",
    "        if row[1]['BraTS21ID_full'] == '00123' and mri_type == 't1w':\n",
    "            continue\n",
    "        if row[1]['BraTS21ID_full'] == '00709' and mri_type == 'flair':\n",
    "            continue\n",
    "        img_dir = row[1][mri_type]\n",
    "        img_files = os.listdir(img_dir)\n",
    "        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n",
    "        mid_point = int(len(img_nums)/2)\n",
    "        start_point = mid_point - max(int(mid_point*0.1), 1)\n",
    "        end_point = mid_point + max(int(mid_point*0.1), 1)\n",
    "        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n",
    "        img_paths = [img_dir+ele for ele in img_names]\n",
    "        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n",
    "        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n",
    "        all_img_files.extend(img_paths)\n",
    "        all_img_labels.extend(img_labels)\n",
    "        all_img_patient_ids.extend(img_patient_ids)\n",
    "\n",
    "    train_val_df = pd.DataFrame({'patient_ids': all_img_patient_ids,\n",
    "                  'labels': all_img_labels,\n",
    "                  'file_paths': all_img_files})\n",
    "\n",
    "    train_val_df['labels'] = train_val_df['labels'].map({1: '1', 0: '0'})\n",
    "    \n",
    "    #stratifiied 90% split on patient_ids and labels  \n",
    "    class_prop= 0.90\n",
    "    \n",
    "    classes_splits  = {}\n",
    "    for i in range(2):\n",
    "        train_val_label_class = train_val_df[train_val_df['labels']==f'{i}']\n",
    "        train_val_list_ids =  list(train_val_label_class['patient_ids'].unique())\n",
    "        train_threshold = math.ceil(class_prop*len(train_val_list_ids))\n",
    "        train_ids = train_val_list_ids[:train_threshold]\n",
    "        val_ids = train_val_list_ids[train_threshold:]\n",
    "        classes_splits[f'train_{i}'] = train_val_label_class[train_val_label_class['patient_ids'].isin(train_ids)]\n",
    "        classes_splits[f'val_{i}'] = val_df = train_val_label_class[train_val_label_class['patient_ids'].isin(val_ids)]\n",
    "        \n",
    "    train_df = pd.concat([classes_splits['train_0'], classes_splits['train_1']], axis=0)\n",
    "    val_df = pd.concat([classes_splits['val_0'], classes_splits['val_1']], axis=0)\n",
    "  \n",
    "    return train_df, val_df\n",
    "    \n",
    "def get_test_dataframe(mri_type):\n",
    "    \n",
    "    all_test_img_files = []\n",
    "    all_test_img_labels = []\n",
    "    all_test_img_patient_ids = []\n",
    "    for row in df_test.iterrows():\n",
    "        img_dir = row[1][mri_type]\n",
    "        img_files = os.listdir(img_dir)\n",
    "        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n",
    "        mid_point = int(len(img_nums)/2)\n",
    "        start_point = mid_point - max(int(mid_point*0.1), 1)\n",
    "        end_point = mid_point + max(int(mid_point*0.1), 1)\n",
    "        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n",
    "        img_paths = [img_dir+ele for ele in img_names]\n",
    "        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n",
    "        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n",
    "        all_test_img_files.extend(img_paths)\n",
    "        all_test_img_labels.extend(img_labels)\n",
    "        all_test_img_patient_ids.extend(img_patient_ids)\n",
    "\n",
    "    test_df = pd.DataFrame({'patient_ids': all_test_img_patient_ids,\n",
    "                  'labels': all_test_img_labels,\n",
    "                  'file_paths': all_test_img_files})\n",
    "    \n",
    "    test_df['labels'] = ['1']*(len(test_df)-1) + ['0'] # workaround for testing data gen\n",
    "    \n",
    "    return test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "animal-oliver",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DCMDataFrameIterator(DataFrameIterator):\n",
    "    def __init__(self, *arg, **kwargs):\n",
    "        self.white_list_formats = ('dcm')\n",
    "        super(DCMDataFrameIterator, self).__init__(*arg, **kwargs)\n",
    "        self.dataframe = kwargs['dataframe']\n",
    "        self.x = self.dataframe[kwargs['x_col']]\n",
    "        self.y = self.dataframe[kwargs['y_col']]\n",
    "        self.color_mode = kwargs['color_mode']\n",
    "        self.target_size = kwargs['target_size']\n",
    "\n",
    "    def _get_batches_of_transformed_samples(self, indices_array):\n",
    "        # get batch of images\n",
    "        batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size, color_mode=self.color_mode)\n",
    "                            for dcm_path in self.x.iloc[indices_array]])\n",
    "\n",
    "        batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8))  # astype because y was passed as str\n",
    "\n",
    "        # transform images\n",
    "        if self.image_data_generator is not None:\n",
    "            for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n",
    "                transform_params = self.image_data_generator.get_random_transform(x.shape)\n",
    "                batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)\n",
    "                # you can change y here as well, eg: in semantic segmentation you want to transform masks as well \n",
    "                # using the same image_data_generator transformations.\n",
    "\n",
    "        return batch_x, batch_y\n",
    "\n",
    "    @staticmethod\n",
    "    def read_dcm_as_array(dcm_path, target_size=(300, 300), color_mode='rgb'):\n",
    "        image_array = pydicom.dcmread(dcm_path).pixel_array\n",
    "        pixels = image_array - np.min(image_array)\n",
    "        pixels = pixels / np.max(pixels)\n",
    "        image_manual_norm = (pixels * 255).astype(np.uint8)\n",
    "        image_array = cv2.resize(image_manual_norm, target_size, interpolation=cv2.INTER_NEAREST)  #this returns a 2d array\n",
    "#         image_array = np.expand_dims(image_array, -1)\n",
    "        if color_mode == 'rgb':\n",
    "            image_array = np.dstack((image_array, np.zeros_like(image_array), np.zeros_like(image_array)))\n",
    "        return image_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "prerequisite-ideal",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 369\n",
    "BATCH_SIZE = 128\n",
    "CLASS_MODE = 'binary'\n",
    "COLOR_MODE = 'rgb'\n",
    "TARGET_SIZE = (300, 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "removed-tennessee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_generators(train_df,val_df, test_df):\n",
    "    train_augmentation_parameters = dict(\n",
    "        rescale=1.0/255,\n",
    "        zoom_range=0.2,\n",
    "        rotation_range=0.2,\n",
    "        fill_mode='nearest',\n",
    "        height_shift_range= 0.1,\n",
    "        width_shift_range=0.1,\n",
    "        horizontal_flip=True,\n",
    "        brightness_range = [0.8, 1.2]\n",
    "    )\n",
    "    \n",
    "    val_augmentation_parameters = dict(\n",
    "        rescale=1.0/255.0\n",
    "    )\n",
    "\n",
    "    test_augmentation_parameters = dict(\n",
    "        rescale=1.0/255.0\n",
    "    )\n",
    "\n",
    "    train_consts = {\n",
    "        'seed': SEED,\n",
    "        'batch_size': BATCH_SIZE,\n",
    "        'class_mode': CLASS_MODE,\n",
    "        'color_mode': COLOR_MODE,\n",
    "        'target_size': TARGET_SIZE,  \n",
    "    }\n",
    "    \n",
    "    val_consts = {\n",
    "    'batch_size': BATCH_SIZE,\n",
    "    'class_mode': CLASS_MODE,\n",
    "    'color_mode': COLOR_MODE,\n",
    "    'target_size': TARGET_SIZE,\n",
    "    'shuffle': False\n",
    "    }\n",
    "\n",
    "    test_consts = {\n",
    "        'batch_size': BATCH_SIZE,\n",
    "        'class_mode': CLASS_MODE,\n",
    "        'color_mode': COLOR_MODE,\n",
    "        'target_size': TARGET_SIZE,\n",
    "        'shuffle': False\n",
    "    }\n",
    "\n",
    "    train_augmenter = ImageDataGenerator(**train_augmentation_parameters)\n",
    "    val_augmenter = ImageDataGenerator(**val_augmentation_parameters)\n",
    "    test_augmenter = ImageDataGenerator(**test_augmentation_parameters)\n",
    "\n",
    "    train_generator = DCMDataFrameIterator(dataframe=train_df,\n",
    "                                 x_col='file_paths',\n",
    "                                 y_col='labels',\n",
    "                                 image_data_generator=train_augmenter,\n",
    "                                 **train_consts)\n",
    "    \n",
    "    val_generator = DCMDataFrameIterator(dataframe=val_df,\n",
    "                                 x_col='file_paths',\n",
    "                                 y_col='labels',\n",
    "                                 image_data_generator=val_augmenter,\n",
    "                                 **val_consts)\n",
    "    \n",
    "    test_generator = DCMDataFrameIterator(dataframe=test_df,\n",
    "                                 x_col='file_paths',\n",
    "                                 y_col='labels',\n",
    "                                 image_data_generator=test_augmenter,\n",
    "                                 **test_consts)\n",
    "    \n",
    "    return train_generator, val_generator, test_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "referenced-basketball",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(weights_path):\n",
    "    \n",
    "    model = EfficientNetB3(include_top=False, weights=weights_path)\n",
    "    \n",
    "    model.trainable = False\n",
    "    \n",
    "    x = GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    top_dropout_rate = 0.4\n",
    "    x = Dropout(top_dropout_rate)(x)\n",
    "    x = Dense(32, activation=\"relu\")(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(top_dropout_rate)(x)\n",
    "    outputs = Dense(1, activation=\"sigmoid\", name=\"pred\")(x)\n",
    "    \n",
    "\n",
    "    # Compile\n",
    "    model = Model(model.inputs, outputs, name=\"EfficientNet\")\n",
    "    \n",
    "\n",
    "    # Compile\n",
    "    optimizer =  tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
    "    \n",
    "    model.compile(optimizer=optimizer, loss=\"binary_crossentropy\", metrics=[\"binary_accuracy\",AUC()])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "surprised-heater",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_filepath = 'best_model.h5'\n",
    "\n",
    "def train_model(model_name, train_generator, val_generator, epochs):\n",
    "    \n",
    "    print('training', model_name)\n",
    "    \n",
    "    model = build_model(\"E:/MyMasterthesis/Models/efficientnet-b3_tf24_imagenet_1000_notop.h5\")\n",
    "    \n",
    "    #callbacks\n",
    "    \n",
    "    checkpoint_cb=ModelCheckpoint(\n",
    "        filepath=checkpoint_filepath,\n",
    "        save_weights_only=False,\n",
    "        monitor='val_loss',\n",
    "        mode='min',\n",
    "        save_best_only=True,\n",
    "        save_freq='epoch',\n",
    "        verbose=1)\n",
    "    \n",
    "    early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                                                  patience=5,\n",
    "                                                  mode='min',\n",
    "                                                  verbose=1,\n",
    "                                                  restore_best_weights=True)\n",
    "\n",
    "    reduce_lr_cb=ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n",
    "                                   patience=2, min_lr=0.00001,\n",
    "                                  verbose=1)\n",
    "\n",
    "    history = model.fit(\n",
    "                        train_generator,\n",
    "                        steps_per_epoch=len(train_generator),\n",
    "                        validation_data=val_generator,\n",
    "                        validation_steps=len(val_generator),\n",
    "                        epochs=epochs,\n",
    "                        workers=2,\n",
    "                        callbacks=[checkpoint_cb, reduce_lr_cb, early_stopping_cb]\n",
    "                        )\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "moved-intake",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 7306 validated image filenames belonging to 2 classes.\n",
      "Found 333 validated image filenames belonging to 2 classes.\n",
      "Found 1119 validated image filenames belonging to 2 classes.\n",
      "training flair\n",
      "Epoch 1/20\n",
      "58/58 [==============================] - 1727s 29s/step - loss: 0.9279 - binary_accuracy: 0.5377 - auc: 0.5443 - val_loss: 0.6816 - val_binary_accuracy: 0.5135 - val_auc: 0.5739\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.68163, saving model to best_model.h5\n",
      "Epoch 2/20\n",
      "58/58 [==============================] - 1665s 28s/step - loss: 0.7600 - binary_accuracy: 0.5856 - auc: 0.6030 - val_loss: 0.6862 - val_binary_accuracy: 0.5225 - val_auc: 0.5531\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.68163\n",
      "Epoch 3/20\n",
      "58/58 [==============================] - 1638s 28s/step - loss: 0.7126 - binary_accuracy: 0.6115 - auc: 0.6274 - val_loss: 0.6782 - val_binary_accuracy: 0.5345 - val_auc: 0.5890\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.68163 to 0.67823, saving model to best_model.h5\n",
      "Epoch 4/20\n",
      "58/58 [==============================] - 1605s 27s/step - loss: 0.6934 - binary_accuracy: 0.6106 - auc: 0.6344 - val_loss: 0.6972 - val_binary_accuracy: 0.5345 - val_auc: 0.5453\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.67823\n",
      "Epoch 5/20\n",
      "58/58 [==============================] - 1629s 28s/step - loss: 0.6768 - binary_accuracy: 0.6140 - auc: 0.6452 - val_loss: 0.6767 - val_binary_accuracy: 0.6036 - val_auc: 0.6144\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.67823 to 0.67670, saving model to best_model.h5\n",
      "Epoch 6/20\n",
      "58/58 [==============================] - 1641s 28s/step - loss: 0.6508 - binary_accuracy: 0.6396 - auc: 0.6658 - val_loss: 0.6868 - val_binary_accuracy: 0.5796 - val_auc: 0.5994\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.67670\n",
      "Epoch 7/20\n",
      "58/58 [==============================] - 1678s 29s/step - loss: 0.6399 - binary_accuracy: 0.6371 - auc: 0.6801 - val_loss: 0.6781 - val_binary_accuracy: 0.6066 - val_auc: 0.6188\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.67670\n",
      "\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 8/20\n",
      "58/58 [==============================] - 1680s 29s/step - loss: 0.6216 - binary_accuracy: 0.6611 - auc: 0.7003 - val_loss: 0.6778 - val_binary_accuracy: 0.6096 - val_auc: 0.6254\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.67670\n",
      "Epoch 9/20\n",
      "58/58 [==============================] - 1673s 29s/step - loss: 0.6323 - binary_accuracy: 0.6376 - auc: 0.6827 - val_loss: 0.6849 - val_binary_accuracy: 0.5796 - val_auc: 0.6022\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.67670\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/20\n",
      "58/58 [==============================] - 1679s 29s/step - loss: 0.6208 - binary_accuracy: 0.6487 - auc: 0.6982 - val_loss: 0.6926 - val_binary_accuracy: 0.5976 - val_auc: 0.6019\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.67670\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00010: early stopping\n",
      "9/9 [==============================] - 236s 25s/step - loss: 0.5799 - binary_accuracy: 0.7131 - auc: 0.2961\n",
      "test loss, test acc, test AUC: [0.5798919796943665, 0.7131367325782776, 0.2960644066333771]\n",
      "Found 7347 validated image filenames belonging to 2 classes.\n",
      "Found 389 validated image filenames belonging to 2 classes.\n",
      "Found 1187 validated image filenames belonging to 2 classes.\n",
      "training t1w\n",
      "Epoch 1/20\n",
      "58/58 [==============================] - 1834s 30s/step - loss: 0.9050 - binary_accuracy: 0.5360 - auc_1: 0.5439 - val_loss: 0.6815 - val_binary_accuracy: 0.5604 - val_auc_1: 0.5875\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.68148, saving model to best_model.h5\n",
      "Epoch 2/20\n",
      "58/58 [==============================] - 1715s 29s/step - loss: 0.7658 - binary_accuracy: 0.5684 - auc_1: 0.5929 - val_loss: 0.6851 - val_binary_accuracy: 0.5219 - val_auc_1: 0.5874\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.68148\n",
      "Epoch 3/20\n",
      "58/58 [==============================] - 5073s 88s/step - loss: 0.7190 - binary_accuracy: 0.5910 - auc_1: 0.6209 - val_loss: 0.6931 - val_binary_accuracy: 0.5039 - val_auc_1: 0.5520\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.68148\n",
      "\n",
      "Epoch 00003: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 4/20\n",
      "58/58 [==============================] - 1964s 33s/step - loss: 0.6980 - binary_accuracy: 0.5936 - auc_1: 0.6258 - val_loss: 0.6932 - val_binary_accuracy: 0.5064 - val_auc_1: 0.5528\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.68148\n",
      "Epoch 5/20\n",
      "58/58 [==============================] - 1968s 34s/step - loss: 0.6899 - binary_accuracy: 0.5981 - auc_1: 0.6360 - val_loss: 0.6921 - val_binary_accuracy: 0.4936 - val_auc_1: 0.5649\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.68148\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 6/20\n",
      "58/58 [==============================] - 1997s 34s/step - loss: 0.6866 - binary_accuracy: 0.5982 - auc_1: 0.6351 - val_loss: 0.6937 - val_binary_accuracy: 0.5141 - val_auc_1: 0.5690\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.68148\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00006: early stopping\n",
      "10/10 [==============================] - 312s 31s/step - loss: 0.6879 - binary_accuracy: 0.5838 - auc_1: 0.4334\n",
      "test loss, test acc, test AUC: [0.6878772974014282, 0.5838247537612915, 0.4333895444869995]\n",
      "Found 9312 validated image filenames belonging to 2 classes.\n",
      "Found 387 validated image filenames belonging to 2 classes.\n",
      "Found 1441 validated image filenames belonging to 2 classes.\n",
      "training t1wce\n",
      "Epoch 1/20\n",
      "73/73 [==============================] - 2933s 39s/step - loss: 0.9027 - binary_accuracy: 0.5227 - auc_2: 0.5359 - val_loss: 0.6783 - val_binary_accuracy: 0.5995 - val_auc_2: 0.6036\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.67827, saving model to best_model.h5\n",
      "Epoch 2/20\n",
      "73/73 [==============================] - 2855s 39s/step - loss: 0.7628 - binary_accuracy: 0.5621 - auc_2: 0.5810 - val_loss: 0.6880 - val_binary_accuracy: 0.5297 - val_auc_2: 0.6097\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.67827\n",
      "Epoch 3/20\n",
      "73/73 [==============================] - 2857s 39s/step - loss: 0.7169 - binary_accuracy: 0.5830 - auc_2: 0.6082 - val_loss: 0.6938 - val_binary_accuracy: 0.5271 - val_auc_2: 0.6009\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.67827\n",
      "\n",
      "Epoch 00003: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 4/20\n",
      "73/73 [==============================] - 2763s 38s/step - loss: 0.6779 - binary_accuracy: 0.6073 - auc_2: 0.6437 - val_loss: 0.7004 - val_binary_accuracy: 0.5090 - val_auc_2: 0.5914\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.67827\n",
      "Epoch 5/20\n",
      "73/73 [==============================] - 2146s 29s/step - loss: 0.6669 - binary_accuracy: 0.6200 - auc_2: 0.6530 - val_loss: 0.7028 - val_binary_accuracy: 0.5090 - val_auc_2: 0.5877\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.67827\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 6/20\n",
      "73/73 [==============================] - 2313s 32s/step - loss: 0.6616 - binary_accuracy: 0.6167 - auc_2: 0.6571 - val_loss: 0.7013 - val_binary_accuracy: 0.5349 - val_auc_2: 0.5916\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.67827\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00006: early stopping\n",
      "12/12 [==============================] - 445s 32s/step - loss: 0.6505 - binary_accuracy: 0.7023 - auc_2: 0.7038\n",
      "test loss, test acc, test AUC: [0.6504726409912109, 0.7022900581359863, 0.7038194537162781]\n",
      "Found 9830 validated image filenames belonging to 2 classes.\n",
      "Found 325 validated image filenames belonging to 2 classes.\n",
      "Found 1463 validated image filenames belonging to 2 classes.\n",
      "training t2w\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-18-dc17bad5b017>:32: RuntimeWarning: invalid value encountered in true_divide\n",
      "  pixels = pixels / np.max(pixels)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77/77 [==============================] - 3223s 39s/step - loss: 0.8921 - binary_accuracy: 0.5384 - auc_3: 0.5399 - val_loss: 0.6705 - val_binary_accuracy: 0.5846 - val_auc_3: 0.5992\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.67051, saving model to best_model.h5\n",
      "Epoch 2/20\n",
      "77/77 [==============================] - 2880s 37s/step - loss: 0.7334 - binary_accuracy: 0.5942 - auc_3: 0.5983 - val_loss: 0.6753 - val_binary_accuracy: 0.5908 - val_auc_3: 0.5678\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.67051\n",
      "Epoch 3/20\n",
      "77/77 [==============================] - 3094s 40s/step - loss: 0.6908 - binary_accuracy: 0.6144 - auc_3: 0.6176 - val_loss: 0.6691 - val_binary_accuracy: 0.5631 - val_auc_3: 0.5992\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.67051 to 0.66910, saving model to best_model.h5\n",
      "Epoch 4/20\n",
      "77/77 [==============================] - 2943s 38s/step - loss: 0.6627 - binary_accuracy: 0.6351 - auc_3: 0.6515 - val_loss: 0.6708 - val_binary_accuracy: 0.6123 - val_auc_3: 0.5987\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.66910\n",
      "Epoch 5/20\n",
      "77/77 [==============================] - 2906s 38s/step - loss: 0.6540 - binary_accuracy: 0.6330 - auc_3: 0.6560 - val_loss: 0.6687 - val_binary_accuracy: 0.6123 - val_auc_3: 0.6116\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.66910 to 0.66874, saving model to best_model.h5\n",
      "Epoch 6/20\n",
      "77/77 [==============================] - 2865s 37s/step - loss: 0.6416 - binary_accuracy: 0.6471 - auc_3: 0.6703 - val_loss: 0.6787 - val_binary_accuracy: 0.6000 - val_auc_3: 0.6082\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.66874\n",
      "Epoch 7/20\n",
      "77/77 [==============================] - 2823s 37s/step - loss: 0.6283 - binary_accuracy: 0.6492 - auc_3: 0.6879 - val_loss: 0.6697 - val_binary_accuracy: 0.5969 - val_auc_3: 0.6117\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.66874\n",
      "\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 8/20\n",
      "77/77 [==============================] - 3307s 43s/step - loss: 0.6180 - binary_accuracy: 0.6570 - auc_3: 0.6980 - val_loss: 0.6753 - val_binary_accuracy: 0.5969 - val_auc_3: 0.6153\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.66874\n",
      "Epoch 9/20\n",
      "77/77 [==============================] - 3813s 49s/step - loss: 0.6117 - binary_accuracy: 0.6671 - auc_3: 0.7046 - val_loss: 0.6763 - val_binary_accuracy: 0.5969 - val_auc_3: 0.6084\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.66874\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/20\n",
      "77/77 [==============================] - 4497s 58s/step - loss: 0.6070 - binary_accuracy: 0.6717 - auc_3: 0.7119 - val_loss: 0.6740 - val_binary_accuracy: 0.5938 - val_auc_3: 0.6063\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.66874\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00010: early stopping\n",
      "12/12 [==============================] - 591s 44s/step - loss: 0.5434 - binary_accuracy: 0.8154 - auc_3: 0.5999\n",
      "test loss, test acc, test AUC: [0.5433711409568787, 0.8154476881027222, 0.5998631715774536]\n",
      "Wall time: 23h 10min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# train a model for each of the mri types and then ensemble predictions\n",
    "all_test_preds = []\n",
    "\n",
    "for mt in ['flair', 't1w', 't1wce', 't2w']:\n",
    "    train_df, val_df = get_train_val_dataframe(mt)\n",
    "    test_df = get_test_dataframe(mt)\n",
    "    train_g, val_g, test_g = get_data_generators(train_df, val_df, test_df)\n",
    "    best_model =  train_model(mt, train_g, val_g, epochs=20)\n",
    "    results = best_model.evaluate(test_g, steps=len(test_g))\n",
    "    print(f\"test loss, test acc, test AUC: {results}\")\n",
    "    test_pred = best_model.predict(test_g, steps=len(test_g))\n",
    "    test_df['pred_y'] = test_pred\n",
    "    # aggregate the predictions on all image for each person (take the most confident prediction out of all image predictions)\n",
    "    mean_pred = test_pred.mean()\n",
    "    test_pred_agg = test_df.groupby('patient_ids').apply(\n",
    "        lambda x: x['pred_y'].max()\n",
    "        if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) \n",
    "        else x['pred_y'].min())\n",
    "    all_test_preds.append(test_pred_agg.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "complete-yukon",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_test_preds = np.array(all_test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "subtle-paintball",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 8., 10.,  9., 14., 13., 10.,  5.,  5.,  9.,  4.]),\n",
       " array([0.36707597, 0.40199806, 0.43692015, 0.47184223, 0.50676432,\n",
       "        0.5416864 , 0.57660849, 0.61153058, 0.64645266, 0.68137475,\n",
       "        0.71629684]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4824 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(all_test_preds.mean(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "boring-angle",
   "metadata": {},
   "outputs": [],
   "source": [
    "subm = pd.read_csv(root_dir+'sample_submission.csv')\n",
    "subm['MGMT_value'] = all_test_preds.mean(0)\n",
    "subm.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cooperative-wales",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BraTS21ID</th>\n",
       "      <th>MGMT_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.593592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>0.467028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>0.488239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>0.618634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>37</td>\n",
       "      <td>0.474315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>826</td>\n",
       "      <td>0.559838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>829</td>\n",
       "      <td>0.523335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>833</td>\n",
       "      <td>0.428486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>997</td>\n",
       "      <td>0.501956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>1006</td>\n",
       "      <td>0.548080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>87 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    BraTS21ID  MGMT_value\n",
       "0           1    0.593592\n",
       "1          13    0.467028\n",
       "2          15    0.488239\n",
       "3          27    0.618634\n",
       "4          37    0.474315\n",
       "..        ...         ...\n",
       "82        826    0.559838\n",
       "83        829    0.523335\n",
       "84        833    0.428486\n",
       "85        997    0.501956\n",
       "86       1006    0.548080\n",
       "\n",
       "[87 rows x 2 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subm"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
