mammo_package.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mammography Example\n",
    "## Reads an initial metadata cohort csv and derives image types, lateralities, spotmags, and extracts ROIs. Finally merging and saving the metadata csv to a given path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# my packaged code\n",
    "from mammo_src.dataframe_processing import *\n",
    "# ROI class\n",
    "from mammo_src.roi_extractions import ROI_extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set cohort number\n",
    "cohort_num = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the cohort dataframe\n",
    "main_df = pd.read_csv('/data/mammo/png_kheiron/kheiron_code/cohort_{}/metadata_anon_cohort_{}.csv'.format(cohort_num, cohort_num), low_memory=False)\n",
    "# alternatively, a simpler dataframe with just the necessary columns can be loaded with load_simple_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# derive image type, lateralities, and separate png_path to folder_path and filename\n",
    "main_df = derive_imgType(main_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# identify special views\n",
    "main_df = get_spotmag_flags(main_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the rooth directories between servers in png_path\n",
    "main_df = correct_root_paths(main_df, '/opt/ssd-data/mammo/png_kheiron/', '/data/mammo/png_kheiron/kheiron_code/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removes the old 'png_path' with the 'corrected_png_path' and renames it to 'png_path'\n",
    "replace_old_png_path(main_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make filename and folder_path dictionary for ROI extraction \n",
    "ss_dict = make_screensave_dict(main_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ROI extraction code\n",
    "ROI = ROI_extraction('/home/jupyter-jjjeon3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n",
      "Scale factor is off, correcting\n"
     ]
    }
   ],
   "source": [
    "# run extractions\n",
    "ROI_coords_ssc, ROI_coords_mammo, ROI_matching_ssc, ROI_matching_mammo = ROI.run_extractions(ss_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge into SSC or Mammo dataframes for mapping back to original metadata\n",
    "#ssc_roi_df = pd.DataFrame({'ROI_coord' : ROI_coords_ssc, 'matching_path' : ROI_matching_ssc})\n",
    "mammo_roi_df = pd.DataFrame({'ROI_coord' : ROI_coords_mammo, 'matching_path' : ROI_matching_mammo})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save in the interim just in case\n",
    "#ssc_roi_df.to_csv('ROI_extracted_SSC_cohort_{}.csv'.format(cohort_num))\n",
    "mammo_roi_df.to_csv('ROI_extracted_Mammo_cohort_{}.csv'.format(cohort_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# will remove this step but need to load the path as a string for the merging to work\n",
    "#ssc_roi_df = pd.read_csv('ROI_extracted_SSC_cohort_{}.csv'.format(cohort_num))\n",
    "mammo_roi_df = pd.read_csv('ROI_extracted_Mammo_cohort_{}.csv'.format(cohort_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map back the ssc coordinates and the mammo coordinates\n",
    "# main_df = match_roi(main_df, ssc_roi_df)\n",
    "main_df = match_roi(main_df, mammo_roi_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the final metadata csv to a path using pd.to_csv(./path/metadata_filename.csv)\n",
    "# load the final metadata csv in the correct ROI coordinates format using: df = read_df(./path/metadata_filename.csv)\n",
    "main_df.to_csv('/data/mammo/png_kheiron/kheiron_code/cohort_{}/metadata_anon_cohort_{}_ROI.csv'.format(cohort_num, cohort_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}