quenn v0.1

.gitignore

@@ -0,0 +1,130 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+# C extensions
+*.so
+.idea
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+net_logs
+logs_backup
+.ipynb_checkpoints
+chk
+data

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Aashis Khanal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,4 @@
+### quenn
+**Qu**ick **N**eural **N**etwork **E**xperimentation
+
+### Please see experiments folder for examples

example/README.md

@@ -0,0 +1,10 @@
+### quenn
+**Qu**ick **N**eural **N**etwork **E**xperimentation
+
+### This document consists of an example on the following datasets:
+* **DRIVE**: Staal, J., Abramoff, M., Niemeijer, M., Viergever, M., and van Ginneken, B. (2004). 
+Ridge based vessel segmentation in color images of the retina.
+IEEE Transactions on Medical Imaging23, 501–509
+* **AV_WIDE**: Estrada,  R.,  Tomasi,  C.,  Schmidler,  S. C.,  and Farsiu,  S. (2015).  
+Tree topology estimation. IEEE Transactions on Pattern Analysis and Machine Intelligence
+37, 1688–1701. doi:10.1109/TPAMI.2014.2592382116

example/classification.py

@@ -0,0 +1,103 @@
+import os
+import random
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torchvision.transforms as tmf
+from PIL import Image as IMG
+
+from quenn.utils.imageutils import Image
+from quenn.core.measurements import Avg, Prf1a
+from quenn.core.nn import QNTrainer, QNDataset
+from example.models import DiskExcNet
+
+sep = os.sep
+
+
+class KernelDataset(QNDataset):
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.get_label = kw.get('label_getter')
+        self.get_mask = kw.get('mask_getter')
+
+    def load_index(self, map_id, file_id, file):
+        self.indices.append([map_id, file_id, file])
+
+    def __getitem__(self, index):
+        map_id, file_id, file = self.indices[index]
+        dt = self.dmap[map_id]
+
+        img_obj = Image()
+        img_obj.load(dt['data_dir'], file)
+        img_obj.load_ground_truth(dt['label_dir'], dt['label_getter'])
+        img_obj.apply_clahe()
+        if self.mode == 'train' and random.uniform(0, 1) <= 0.5:
+            img_obj.array = np.flip(img_obj.array, 0)
+            img_obj.ground_truth = np.flip(img_obj.ground_truth, 0)
+
+        if self.mode == 'train' and random.uniform(0, 1) <= 0.5:
+            img_obj.array = np.flip(img_obj.array, 1)
+            img_obj.ground_truth = np.flip(img_obj.ground_truth, 1)
+
+        img_tensor = self.transforms(IMG.fromarray(img_obj.array))
+
+        gt = img_obj.ground_truth
+        if len(gt.shape) > 2:
+            gt = gt[:, :, 0]
+
+        gt = self.transforms(IMG.fromarray(gt))
+        gt[gt == 255] = 1
+        return {'indices': self.indices[index], 'input': img_tensor, 'label': gt.squeeze()}
+
+    @property
+    def transforms(self):
+        return tmf.Compose(
+            [tmf.Resize((128, 128)), tmf.ToTensor()])
+
+
+class KernelTrainer(QNTrainer):
+    def __init__(self, args, **kw):
+        super().__init__(args, **kw)
+
+    def _init_nn(self):
+        self.nn['model'] = DiskExcNet(self.args['num_channel'], self.args['num_class'], r=self.args['model_scale'])
+
+    def iteration(self, batch):
+        inputs = batch['input'].to(self.nn['device']).float()
+        labels = batch['label'].to(self.nn['device']).long()
+
+        out = self.nn['model'](inputs)
+        loss = F.cross_entropy(out, labels)
+        out = F.log_softmax(out, 1)
+
+        _, pred = torch.max(out, 1)
+        sc = self.new_metrics()
+        sc.add(pred, labels)
+
+        avg = Avg()
+        avg.add(loss.item(), len(inputs))
+
+        return {'loss': loss, 'avg_loss': avg, 'output': out, 'scores': sc, 'predictions': pred}
+
+    def save_predictions(self, accumulator):
+        dataset_name = list(accumulator[0].dmap.keys()).pop()
+        file = accumulator[1][0]['indices'][2][0].split('.')[0]
+        out = accumulator[1][1]['output']
+        img = out[:, 1, :, :].cpu().numpy() * 255
+        img = np.array(img.squeeze(), dtype=np.uint8)
+        IMG.fromarray(img).save(self.cache['log_dir'] + sep + dataset_name + '_' + file + '.png')
+
+    def new_metrics(self):
+        return Prf1a()
+
+    def reset_dataset_cache(self):
+        self.cache['global_test_score'] = []
+        self.cache['monitor_metrics'] = 'f1'
+        self.cache['score_direction'] = 'maximize'
+
+    def reset_fold_cache(self):
+        self.cache['training_log'] = ['Loss,Precision,Recall,F1,Accuracy']
+        self.cache['validation_log'] = ['Loss,Precision,Recall,F1,Accuracy']
+        self.cache['test_score'] = ['Split,Precision,Recall,F1,Accuracy']
+        self.cache['best_score'] = 0.0

example/datasets/AV-WIDE/splits/AV-WIDE_0.json

@@ -0,0 +1 @@
+{"train": ["wide_image_01.png", "wide_image_29.png", "wide_image_09.png", "wide_image_18.png", "wide_image_10.png", "wide_image_05.png", "wide_image_21.png", "wide_image_07.png", "wide_image_20.png", "wide_image_23.png", "wide_image_13.png", "wide_image_14.png", "wide_image_22.png", "wide_image_24.png", "wide_image_28.png", "wide_image_15.png", "wide_image_16.png", "wide_image_02.png"], "validation": ["wide_image_08.png", "wide_image_19.png", "wide_image_30.png", "wide_image_26.png", "wide_image_27.png", "wide_image_12.png"], "test": ["wide_image_03.png", "wide_image_11.png", "wide_image_17.png", "wide_image_06.png", "wide_image_25.png", "wide_image_04.png"]}

example/datasets/AV-WIDE/splits/AV-WIDE_1.json

@@ -0,0 +1 @@
+{"train": ["wide_image_03.png", "wide_image_11.png", "wide_image_17.png", "wide_image_06.png", "wide_image_25.png", "wide_image_04.png", "wide_image_21.png", "wide_image_07.png", "wide_image_20.png", "wide_image_23.png", "wide_image_13.png", "wide_image_14.png", "wide_image_22.png", "wide_image_24.png", "wide_image_28.png", "wide_image_15.png", "wide_image_16.png", "wide_image_02.png"], "validation": ["wide_image_01.png", "wide_image_29.png", "wide_image_09.png", "wide_image_18.png", "wide_image_10.png", "wide_image_05.png"], "test": ["wide_image_08.png", "wide_image_19.png", "wide_image_30.png", "wide_image_26.png", "wide_image_27.png", "wide_image_12.png"]}

example/datasets/AV-WIDE/splits/AV-WIDE_2.json

@@ -0,0 +1 @@
+{"train": ["wide_image_03.png", "wide_image_11.png", "wide_image_17.png", "wide_image_06.png", "wide_image_25.png", "wide_image_04.png", "wide_image_08.png", "wide_image_19.png", "wide_image_30.png", "wide_image_26.png", "wide_image_27.png", "wide_image_12.png", "wide_image_22.png", "wide_image_24.png", "wide_image_28.png", "wide_image_15.png", "wide_image_16.png", "wide_image_02.png"], "validation": ["wide_image_21.png", "wide_image_07.png", "wide_image_20.png", "wide_image_23.png", "wide_image_13.png", "wide_image_14.png"], "test": ["wide_image_01.png", "wide_image_29.png", "wide_image_09.png", "wide_image_18.png", "wide_image_10.png", "wide_image_05.png"]}

example/datasets/AV-WIDE/splits/AV-WIDE_3.json

@@ -0,0 +1 @@
+{"train": ["wide_image_03.png", "wide_image_11.png", "wide_image_17.png", "wide_image_06.png", "wide_image_25.png", "wide_image_04.png", "wide_image_08.png", "wide_image_19.png", "wide_image_30.png", "wide_image_26.png", "wide_image_27.png", "wide_image_12.png", "wide_image_01.png", "wide_image_29.png", "wide_image_09.png", "wide_image_18.png", "wide_image_10.png", "wide_image_05.png"], "validation": ["wide_image_22.png", "wide_image_24.png", "wide_image_28.png", "wide_image_15.png", "wide_image_16.png", "wide_image_02.png"], "test": ["wide_image_21.png", "wide_image_07.png", "wide_image_20.png", "wide_image_23.png", "wide_image_13.png", "wide_image_14.png"]}

example/datasets/AV-WIDE/splits/AV-WIDE_4.json

@@ -0,0 +1 @@
+{"train": ["wide_image_08.png", "wide_image_19.png", "wide_image_30.png", "wide_image_26.png", "wide_image_27.png", "wide_image_12.png", "wide_image_01.png", "wide_image_29.png", "wide_image_09.png", "wide_image_18.png", "wide_image_10.png", "wide_image_05.png", "wide_image_21.png", "wide_image_07.png", "wide_image_20.png", "wide_image_23.png", "wide_image_13.png", "wide_image_14.png"], "validation": ["wide_image_03.png", "wide_image_11.png", "wide_image_17.png", "wide_image_06.png", "wide_image_25.png", "wide_image_04.png"], "test": ["wide_image_22.png", "wide_image_24.png", "wide_image_28.png", "wide_image_15.png", "wide_image_16.png", "wide_image_02.png"]}

example/datasets/AV-WIDE/splits/AV-WIDE_5.json

@@ -0,0 +1 @@
+{"train": ["wide_image_29.png", "wide_image_28.png", "wide_image_19.png", "wide_image_23.png", "wide_image_05.png", "wide_image_09.png", "wide_image_24.png", "wide_image_03.png", "wide_image_06.png", "wide_image_17.png", "wide_image_18.png", "wide_image_21.png", "wide_image_13.png", "wide_image_25.png", "wide_image_30.png", "wide_image_10.png", "wide_image_22.png", "wide_image_14.png", "wide_image_08.png", "wide_image_04.png"], "validation": ["wide_image_27.png", "wide_image_15.png", "wide_image_16.png", "wide_image_12.png", "wide_image_26.png"], "test": ["wide_image_07.png", "wide_image_01.png", "wide_image_11.png", "wide_image_02.png", "wide_image_20.png"]}

example/datasets/DRIVE/OD_Segmentation/Untitled.ipynb

@@ -0,0 +1,71 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from PIL import Image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "arr = np.array(Image.open(\"04_test_gt.tif\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([  0, 255], dtype=uint8)"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.unique(arr)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}