{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5-fold EfficientNet-B3 model using Fastai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 228
    },
    "colab_type": "code",
    "id": "-MzdfcwC5qDr",
    "outputId": "c729d35f-2903-4596-b63a-2fbae9830c53"
   },
   "outputs": [],
   "source": [
    "!rm -r ../root/.kaggle\n",
    "!mkdir ../root/.kaggle\n",
    "!cp kaggle.json ../root/.kaggle\n",
    "!chmod 600 ../root/.kaggle/kaggle.json\n",
    "!rm -r sample_data\n",
    "!pip install efficientnet-pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing packages for our experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xJbKpakU6S6r"
   },
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.vision import *\n",
    "from fastai.vision.models import efficientnet\n",
    "from fastai.utils.ipython import *\n",
    "from fastai.callbacks.tracker import SaveModelCallback\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up k-fold training strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "id": "WtLde_oj6Uqv",
    "outputId": "23c0eaab-4401-4379-f0e9-087adc4b4298"
   },
   "outputs": [],
   "source": [
    "path = Path(\".\")\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 312
    },
    "colab_type": "code",
    "id": "e0L5bOab7Li0",
    "outputId": "14c38074-52aa-4587-d0c5-b92af95f2d6c"
   },
   "outputs": [],
   "source": [
    "data_init = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_none()\n",
    "            .label_from_folder())\n",
    "data_init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qg07Zti59Sg5",
    "outputId": "b83370a6-d5d4-4911-cf72-22220e35c8ad"
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n",
    "skf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-r38w3l8ExI1"
   },
   "outputs": [],
   "source": [
    "!mkdir sipakmed_wsi_pap_smear/models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Btupbe9q9V2M"
   },
   "outputs": [],
   "source": [
    "def model_callback(model, model_name):\n",
    "  return [SaveModelCallback(model, every=\"improvement\", monitor=\"accuracy\", name=model_name)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v5Xkj0BSxLkz"
   },
   "outputs": [],
   "source": [
    "tfms = get_transforms(flip_vert=True, max_warp=0.0, max_rotate=60.0, max_zoom=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "id": "4ec4oK-ongtj",
    "outputId": "d1033339-20a1-4c13-9955-f4211810f5f3"
   },
   "outputs": [],
   "source": [
    "our_metrics = [accuracy, Precision(average=\"macro\"), Recall(average=\"macro\"), FBeta(average=\"macro\"), KappaScore(weights=\"quadratic\")]\n",
    "our_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3BrARh-QmS2K"
   },
   "source": [
    "## K-Fold Training Process\n",
    "* Loop through fold indices\n",
    "* Load the fold data\n",
    "* Initialize the learner with the fold data (with freezing)\n",
    "* Train the learner\n",
    "* Fine-tune the learner\n",
    "* Repeat for different folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FSqIh31zlw7m"
   },
   "outputs": [],
   "source": [
    "idxs = [[train_idx, val_idx] for train_idx, val_idx in skf.split(data_init.x.items, data_init.y.items)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pBXCI83JNCdo"
   },
   "outputs": [],
   "source": [
    "fold_idxs = idxs[0]\n",
    "fold_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_by_idxs(fold_idxs[0], fold_idxs[1])\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121,
     "referenced_widgets": [
      "da5e79f97b674cbdad85fe0c151c6f8d",
      "7044c21d0cda49e58bd413dcc43e3593",
      "17dbe952e63b43b0bc27e42731408c4e",
      "fc74b19438ab4299960df324b1b44df2",
      "e667774d545d4192ad6f77102de6d469",
      "b81ae61c978743559243dc3bc35b3ee3",
      "7298a50d8b9b46cdb1ccf79647e1f894",
      "efb2501f16c2430187c40ca871b6c88b"
     ]
    },
    "colab_type": "code",
    "id": "pQp0iKJ-NCg8",
    "outputId": "9a98a7e3-2be2-48d8-a2da-f4671b15c101"
   },
   "outputs": [],
   "source": [
    "learner = Learner(fold_data, efficientnet.EfficientNetB3(fold_data), metrics=our_metrics).to_fp16()\n",
    "learner = learner.split([learner.model._conv_stem, learner.model._blocks, learner.model._conv_head])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "v6Fkw6SYPDAw",
    "outputId": "e84c0590-2682-4b62-eb4a-83c1c83f7a2a"
   },
   "outputs": [],
   "source": [
    "# Re-run this cell if no graph generated\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "JCDxaeVaPGY_",
    "outputId": "f5a92f06-7e18-4d64-da1c-26b80cd07a71"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate with the least loss value and has a good downward slope.\n",
    "learner.fit_one_cycle(30, max_lr=slice(2e-03), callbacks=model_callback(learner, \"best-effb3-fold1-stage1\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "Jiwzs8IuPYKa",
    "outputId": "69db6ba3-2cdd-4b6c-b022-2bff623c0268"
   },
   "outputs": [],
   "source": [
    "# Re-run this cell if no graph generated\n",
    "learner.load(\"best-effb3-fold1-stage1\")\n",
    "learner.unfreeze()\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "dZLPVIEESvdr",
    "outputId": "86f822a0-d002-4080-d0f8-c2b066d54b99"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate with the least loss value and has appropriate downward slope.\n",
    "learner.fit_one_cycle(30, max_lr=slice(2e-06), callbacks=model_callback(learner, \"best-effb3-fold1-stage2\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Second Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CilAWIJBTAEl"
   },
   "outputs": [],
   "source": [
    "fold_idxs = idxs[1]\n",
    "fold_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_by_idxs(fold_idxs[0], fold_idxs[1])\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "13j-dSNuWitL",
    "outputId": "f6e0a3c3-edc7-4a64-d3a1-4e253c9324c3"
   },
   "outputs": [],
   "source": [
    "# Transferring weights of our model from earlier fold to the new fold\n",
    "# Re-run this cell if no graph generated.\n",
    "learner.load(\"best-effb3-fold1-stage2\")\n",
    "learner.data = fold_data\n",
    "learner.freeze()\n",
    "learner = to_fp16(learner)\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "79lTkQv2Wr9u",
    "outputId": "35bb90a2-24a9-4058-8734-c22cfaf46a6a"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope.\n",
    "learner.fit_one_cycle(30, max_lr=slice(3e-05), callbacks=model_callback(learner, \"best-effb3-fold2-stage1\"))\n",
    "learner.save(\"last-effb3-fold2-stage1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "aB1A8s4uXawp",
    "outputId": "a1383a68-4097-498b-9604-e0d34ed57bad"
   },
   "outputs": [],
   "source": [
    "# Re-run this cell if no graph generated.\n",
    "learner.load(\"last-effb3-fold2-stage1\")\n",
    "learner.unfreeze()\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "0y8UaU8Xa312",
    "outputId": "39320492-1ecf-4661-a9af-026ee626c2f5"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope.\n",
    "learner.fit_one_cycle(30, max_lr=slice(7e-07, 7e-07), callbacks=model_callback(learner, \"best-effb3-fold2-stage2\"))\n",
    "learner.save(\"last-effb3-fold2-stage2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Third Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nR8Bwe1TenBv"
   },
   "outputs": [],
   "source": [
    "fold_idxs = idxs[2]\n",
    "fold_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_by_idxs(fold_idxs[0], fold_idxs[1])\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "U2-9q3-NbNIR",
    "outputId": "64bfc0d9-5a72-4026-c131-f37f2504ea64"
   },
   "outputs": [],
   "source": [
    "# Transferring weights of our model from earlier fold to the new fold\n",
    "# Re-run this cell if no graph generated.\n",
    "\n",
    "learner.purge()\n",
    "learner.load(\"best-effb3-fold2-stage2\")\n",
    "learner.data = fold_data\n",
    "learner.freeze()\n",
    "learner = to_fp16(learner)\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "4aS-gWGtes2j",
    "outputId": "0aafc8f4-e698-4ec9-b4ea-07bc4c94db2c"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope.\n",
    "learner.fit_one_cycle(30, max_lr=slice(5e-06, 6e-06), callbacks=model_callback(learner, \"best-effb3-fold3-stage1\"))\n",
    "learner.save(\"last-effb3-fold3-stage1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "colab_type": "code",
    "id": "WQjTXqV9fPW1",
    "outputId": "3f5a605d-0c96-49f2-8724-f1225e7818e7"
   },
   "outputs": [],
   "source": [
    "# Re-run this cell if no graph generated.\n",
    "learner.load(\"last-effb3-fold3-stage1\")\n",
    "learner.unfreeze()\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 694
    },
    "colab_type": "code",
    "id": "EGL9-RX6ibhO",
    "outputId": "533414c5-fe85-4a54-e9ba-ab978ceffae2"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope\n",
    "learner.fit_one_cycle(20, max_lr=slice(7e-07), callbacks=model_callback(learner, \"best-effb3-fold3-stage2\"))\n",
    "learner.save(\"last-effb3-fold3-stage2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fourth Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hDnI10kauLg6"
   },
   "outputs": [],
   "source": [
    "fold_idxs = idxs[3]\n",
    "fold_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_by_idxs(fold_idxs[0], fold_idxs[1])\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "xAnPHJx5uLjk",
    "outputId": "1de41abc-4409-47b9-f7bf-981a523cc953"
   },
   "outputs": [],
   "source": [
    "# Transferring weights of our model from earlier fold to the new fold\n",
    "# Re-run this cell if graph not generated.\n",
    "\n",
    "learner.purge()\n",
    "learner.load(\"last-effb3-fold3-stage2\")\n",
    "learner.data = fold_data\n",
    "learner.freeze()\n",
    "learner = to_fp16(learner)\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 711
    },
    "colab_type": "code",
    "id": "tA-lycWcuLmP",
    "outputId": "d4407a03-bbae-4b4c-a60f-665021e423b5"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope\n",
    "learner.fit_one_cycle(20, max_lr=slice(1.5e-05), callbacks=model_callback(learner, \"best-effb3-fold4-stage1\"))\n",
    "learner.save(\"last-effb3-fold4-stage1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 410
    },
    "colab_type": "code",
    "id": "kJ6qODL7uLop",
    "outputId": "6a0f5df0-9638-447b-a8da-50afe22dbf48"
   },
   "outputs": [],
   "source": [
    "# Re-run this graph if not generated\n",
    "learner.load(\"best-effb3-fold4-stage1\")\n",
    "learner.unfreeze()\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 711
    },
    "colab_type": "code",
    "id": "nXZEwLUBuLrg",
    "outputId": "6c388dcf-2865-48a1-ac16-15467dcd5be9"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope\n",
    "learner.fit_one_cycle(20, max_lr=slice(2e-06), callbacks=model_callback(learner, \"best-effb3-fold4-stage2\"))\n",
    "learner.save(\"last-effb3-fold4-stage2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fifth Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IHgOKxe5RBhY"
   },
   "outputs": [],
   "source": [
    "fold_idxs = idxs[4]\n",
    "fold_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_by_idxs(fold_idxs[0], fold_idxs[1])\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "m9-l0Ba2uLuH",
    "outputId": "93a5ff21-fe54-48f1-a71f-4ee0ac5a059d"
   },
   "outputs": [],
   "source": [
    "# Transferring weights of our model from earlier fold to the new fold\n",
    "# Re-run this cell if graph not generated.\n",
    "\n",
    "learner.load(\"last-effb3-fold4-stage2\")\n",
    "learner.data = fold_data\n",
    "learner.freeze()\n",
    "learner = to_fp16(learner)\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 694
    },
    "colab_type": "code",
    "id": "JKu6eWhQuLww",
    "outputId": "44dec05f-d386-4c7d-da37-35d773b1d761"
   },
   "outputs": [],
   "source": [
    "# Choose learning rate which has the least loss and has appropriate downward slope\n",
    "learner.fit_one_cycle(20, max_lr=slice(7.5e-07), callbacks=model_callback(learner, \"best-effb3-fold5-stage1\"))\n",
    "learner.save(\"last-effb3-fold5-stage1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "colab_type": "code",
    "id": "HTO_JXU8uL6h",
    "outputId": "b14da279-1bc1-41d5-f873-25806b3f347b"
   },
   "outputs": [],
   "source": [
    "learner.purge()\n",
    "learner.load(\"last-effb3-fold5-stage1\")\n",
    "learner.unfreeze()\n",
    "learner.lr_find()\n",
    "learner.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 694
    },
    "colab_type": "code",
    "id": "9MxQPR8MuLef",
    "outputId": "4b2988ce-1853-46e2-836f-fdc7c3c0167b"
   },
   "outputs": [],
   "source": [
    "learner.fit_one_cycle(20, max_lr=slice(1e-06, 1e-06), callbacks=model_callback(learner, \"best-effb3-fold5-stage2\"))\n",
    "learner.save(\"last-effb3-fold5-stage2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wjEuRmhiF5le"
   },
   "source": [
    "# Model Validation after k-fold training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 312
    },
    "colab_type": "code",
    "id": "J7JRwtsKfOu3",
    "outputId": "80de1dc7-b1db-409b-fa09-73faa2f8e465"
   },
   "outputs": [],
   "source": [
    "all_data = (ImageList.from_folder(path/\"sipakmed_wsi_pap_smear\")\n",
    "            .split_none()\n",
    "            .label_from_folder()\n",
    "            .transform(tfms, size=224)\n",
    "            .databunch(bs=16)\n",
    "            .normalize(imagenet_stats))\n",
    "all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "eDwQVd3SbTxT",
    "outputId": "413037e5-a893-401b-f363-7a2f0454812f"
   },
   "outputs": [],
   "source": [
    "learner.load(\"last-effb3-fold5-stage2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UfNkhGWXej-0"
   },
   "outputs": [],
   "source": [
    "all_learner = to_fp32(learner)\n",
    "all_learner.data = all_data\n",
    "all_learner.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "id": "6XWhajuYjRzU",
    "outputId": "31b29059-8e78-432e-c44c-6cf9b638bb7f"
   },
   "outputs": [],
   "source": [
    "interp = ClassificationInterpretation.from_learner(all_learner, ds_type=DatasetType.Train)\n",
    "interp.plot_confusion_matrix(figsize=(7, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "id": "XBEf5WhVktFJ",
    "outputId": "40e8f31b-169a-4e52-a030-0addd13d077b"
   },
   "outputs": [],
   "source": [
    "interp.plot_confusion_matrix(figsize=(7, 7), normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "WPXVp7wMjdFO",
    "outputId": "d6f70c03-c1c2-470c-9195-ee401e536158"
   },
   "outputs": [],
   "source": [
    "val_learner = to_fp32(all_learner)\n",
    "test_mets = [accuracy, Precision(average=\"macro\"), Recall(average=\"macro\"), FBeta(average=\"macro\"), KappaScore(weights=\"quadratic\")]\n",
    "val_learner.validate(dl=all_data.train_dl, metrics=test_mets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dx2g0jsRlTZF"
   },
   "outputs": [],
   "source": [
    "all_learner.export(\"5fold-effb3-new.pkl\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Sipakmed K-Fold CV in fastai.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "17dbe952e63b43b0bc27e42731408c4e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b81ae61c978743559243dc3bc35b3ee3",
      "max": 49388949,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e667774d545d4192ad6f77102de6d469",
      "value": 49388949
     }
    },
    "7044c21d0cda49e58bd413dcc43e3593": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7298a50d8b9b46cdb1ccf79647e1f894": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b81ae61c978743559243dc3bc35b3ee3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "da5e79f97b674cbdad85fe0c151c6f8d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_17dbe952e63b43b0bc27e42731408c4e",
       "IPY_MODEL_fc74b19438ab4299960df324b1b44df2"
      ],
      "layout": "IPY_MODEL_7044c21d0cda49e58bd413dcc43e3593"
     }
    },
    "e667774d545d4192ad6f77102de6d469": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "efb2501f16c2430187c40ca871b6c88b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fc74b19438ab4299960df324b1b44df2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_efb2501f16c2430187c40ca871b6c88b",
      "placeholder": "​",
      "style": "IPY_MODEL_7298a50d8b9b46cdb1ccf79647e1f894",
      "value": "100% 47.1M/47.1M [00:00<00:00, 60.2MB/s]"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}