optimize threshold for ensembles

baseline_ensembles.py

@@ -0,0 +1,125 @@
+import cv2
+import numpy as np
+import torch.nn as nn
+import torch
+from torch.utils.data import DataLoader
+from torch.autograd import Variable
+from data.kgdataset import KgForestDataset, toTensor
+from torchvision.transforms import Normalize, Compose, Lambda
+from planet_models.resnet_planet import resnet18_planet, resnet34_planet, resnet50_planet
+from planet_models.densenet_planet import densenet161, densenet121, densenet169
+from util import predict
+
+
+def default(imgs):
+    return imgs
+
+
+def rotate90(imgs):
+    for index, img in enumerate(imgs):
+        imgs[index] = cv2.transpose(img, (1, 0, 2))
+    return imgs
+
+
+def rotate180(imgs):
+    for index, img in enumerate(imgs):
+        imgs[index] = cv2.flip(img, -1)
+    return imgs
+
+
+def rotate270(imgs):
+    for index, img in enumerate(imgs):
+        img = cv2.transpose(img, (1, 0, 2))
+        imgs[index] = cv2.flip(img, -1)
+    return imgs
+
+
+def horizontalFlip(imgs):
+    for index, img in enumerate(imgs):
+        img = cv2.flip(img, 1)
+        imgs[index] = img
+    return imgs
+
+
+def verticalFlip(imgs):
+    for index, img in enumerate(imgs):
+        img = cv2.flip(img, 0)
+        imgs[index] = img
+    return imgs
+
+
+mean = [0.31151703, 0.34061992, 0.29885209]
+std = [0.16730586, 0.14391145, 0.13747531]
+transforms = [default, rotate90, rotate180, rotate270, verticalFlip, horizontalFlip]
+models = [resnet18_planet, resnet34_planet, resnet50_planet, densenet121, densenet161, densenet169]
+
+
+
+
+# if __name__ == '__main__':
+#     img = cv2.imread('images.jpeg')
+#     img = cv2.resize(img, (256, 256))
+#     img = np.expand_dims(img, axis=0)
+#     rotation90 = rotate90(img.copy())[0]
+#     rotation180 = rotate180(img.copy())[0]
+#     rotation270 = rotate270(img.copy())[0]
+#     vertical = verticalFlip(img.copy())[0]
+#     horizontal = horizontalFlip(img.copy())[0]
+#     cv2.imshow('original', img[0])
+#     cv2.imshow('90', rotation90)
+#     cv2.imshow('180', rotation180)
+#     cv2.imshow('270', rotation270)
+#     cv2.imshow('vertical', vertical)
+#     cv2.imshow('horizontal', horizontal)
+#
+#     cv2.waitKey()
+
+
+# save probabilities to files for debug
+def probs(dataloader):
+    """
+    returns a numpy array of probabilities (n_transforms, n_models, n_imgs, 17)
+    use transforms to find the best threshold
+    use models to do ensemble method
+    """
+    n_transforms = len(transforms)
+    n_models = len(models)
+    n_imgs = dataloader.dataset.num
+    imgs = dataloader.dataset.images.copy()
+    probabilities = np.empty(n_transforms, n_models, n_imgs, 17)
+    for t_idx, transform in enumerate(transforms):
+        t_name = str(transform).split()[1]
+        dataloader.dataset.images = transform(imgs)
+        for m_idx, model in enumerate(models):
+            name = str(model).split()[1]
+            net = model().cuda()
+            net = nn.DataParallel(net)
+            net = net.load_state_dict(torch.load('models/{}.pth'.format(name)))
+
+            # predict
+            m_predictions = predict(net, dataloader)
+
+            # save
+            np.savetxt(X=m_predictions, fname='probs/{}_{}.txt'.format(t_name, name))
+            probabilities[t_idx, m_idx] = m_predictions
+    return probabilities
+
+# average the results from [verticalFlip, horizontalFlip, transpose]
+
+# optimize the results
+
+
+if __name__ == '__main__':
+    validation = KgForestDataset(
+        split='validation-3000',
+        transform=Compose(
+            [
+                Lambda(lambda x: toTensor(x)),
+                Normalize(mean=mean, std=std)
+            ]
+        ),
+        height=256,
+        width=256
+    )
+    valid_dataloader = DataLoader(validation, batch_size=512, shuffle=False)
+    print(probs(valid_dataloader))

data/kgdataset.py

@@ -98,14 +98,14 @@ def __len__(self):
         return len(self.images)
 
 
+################################# Transformations begin here ################################
 def randomVerticalFlip(img, u=0.5):
     if random.random() < u:
         img = cv2.flip(img, 0)  #np.flipud(img)  #cv2.flip(img,0) ##up-down
     return img
 
 
 def randomHorizontalFlip(img, u=0.5):
-    shape=img.shape
     if random.random() < u:
         img = cv2.flip(img,1)  #np.fliplr(img)  #cv2.flip(img,1) ##left-right
     return img
@@ -210,7 +210,6 @@ def randomShiftScaleRotate(img, u=0.5, shift_limit=4, scale_limit=0.1, rotate_li
         ss = math.sin(angle/180*math.pi)*(scale)
         rotate_matrix = np.array([ [cc,-ss], [ss,cc] ])
 
-
         box0 = np.array([ [0,0], [width,0],  [width,height], [0,height], ])
         box1 = box0 - np.array([width/2,height/2])
         box1 = np.dot(box1,rotate_matrix.T) + np.array([width/2+dx,height/2+dy])
@@ -223,7 +222,6 @@ def randomShiftScaleRotate(img, u=0.5, shift_limit=4, scale_limit=0.1, rotate_li
     return img
 
 
-
 def cropCenter(img, height, width):
 
     h,w,c = img.shape

test.py

@@ -1,20 +1,17 @@
-import pandas as pd
 import torch.nn as nn
 from torch.nn import functional as F
 from datasets import mean, std
-from labels import *
 from planet_models.densenet_planet import densenet121, densenet161, densenet169
-from trainers.train_simplenet import evaluate
 from torchvision.transforms import Compose, Normalize, Lambda
 import numpy as np
 from torch.autograd import Variable
 import torch
 from data.kgdataset import toTensor, KgForestDataset
 from torch.utils.data.dataloader import DataLoader
-from util import pred_csv
+from util import pred_csv, predict
 
 
-BEST_THRESHOLD = np.array([ 0.2071, 0.1986, 0.1296, 0.0363, 0.2355 , 0.1766, 0.2255, 0.257, 0.1922,
+BEST_THRESHOLD = np.array([0.2071, 0.1986, 0.1296, 0.0363, 0.2355 , 0.1766, 0.2255, 0.257, 0.1922,
                             0.1462, 0.2676, 0.0931, 0.213, 0.1041, 0.2606, 0.2872, 0.151])
 
 
@@ -31,38 +28,20 @@ def test():
     )
 
     test_loader = DataLoader(dataset, batch_size=512, shuffle=False, pin_memory=True)
-
-    probs = np.empty((61191, 17))
-    current = 0
-    for batch_idx, (images, im_ids) in enumerate(test_loader):
-        num = images.size(0)
-        previous = current
-        current = previous + num
-        logits = net(Variable(images.cuda(), volatile=True))
-        prob = F.sigmoid(logits)
-        probs[previous:current, :] = prob.data.cpu().numpy()
-        print('Batch Index ', batch_idx)
+    probs = predict(net, test_loader)
+
+    # probs = np.empty((61191, 17))
+    # current = 0
+    # for batch_idx, (images, im_ids) in enumerate(test_loader):
+    #     num = images.size(0)
+    #     previous = current
+    #     current = previous + num
+    #     logits = net(Variable(images.cuda(), volatile=True))
+    #     prob = F.sigmoid(logits)
+    #     probs[previous:current, :] = prob.data.cpu().numpy()
+    #     print('Batch Index ', batch_idx)
 
     pred_csv(probs, name='densenet161', threshold=BEST_THRESHOLD)
-        # result = evaluate(model, images)
-        # result = F.sigmoid(result)
-        # result = result.data.cpu().numpy()
-    #     probs.append(prob.data.cpu().numpy())
-    #     for r, id in zip(result, im_ids):
-    #         label = np.zeros_like(r)
-    #         for i in range(17):
-    #             label[i] = (r[i] > BEST_THRESHOLD[i]).astype(np.int)
-    #         label = np.where(label == 1)[0]
-    #         labels = [idx_to_label[index] for index in label]
-    #         if len(r) == 0:
-    #             print('id', id)
-    #             print('r', r)
-    #         imid_to_label[id] = sorted(labels)
-    #     print('Batch Index {}'.format(batch_idx))
-    # sample_submission = pd.read_csv('/media/jxu7/BACK-UP/Data/AmazonPlanet/sample_submission.csv')
-    # for key, value in imid_to_label.items():
-    #     sample_submission.loc[sample_submission['image_name'] == key,'tags'] = ' '.join(str(x) for x in value)
-    # sample_submission.to_csv('submissions/%s.csv' % name, index=False)
 
 
 if __name__ == '__main__':

trainers/optimize_threshold.py

@@ -3,7 +3,6 @@
 from torch.autograd import Variable
 from planet_models.simplenet_v2 import *
 from datasets import *
-from trainers.train_simplenet import evaluate
 import torch.nn.functional as F
 from util import f2_score
 from data.kgdataset import KgForestDataset, randomTranspose, randomFlip, toTensor

util.py

@@ -19,6 +19,21 @@ def idx_name():
     return {idx: name for idx, name in enumerate(CLASS_NAMES)}
 
 
+def predict(net, dataloader):
+    num = dataloader.dataset.num
+    probs = np.empty(num, 17)
+    current = 0
+    for batch_idx, (images, im_ids) in enumerate(dataloader):
+        num = images.size(0)
+        previous = current
+        current = previous + num
+        logits = net(Variable(images.cuda(), volatile=True))
+        prob = F.sigmoid(logits)
+        probs[previous:current, :] = prob.data.cpu().numpy()
+        print('Batch Index ', batch_idx)
+    return probs
+
+
 def pred_csv(predictions, threshold, name):
     """
     predictions: numpy array of predicted probabilities