Skip to content

Commit

Permalink
Merge pull request #3 from phborba/dev_new_features
Browse files Browse the repository at this point in the history 
Dev new features
  • Loading branch information
@phborba
phborba authored Sep 30, 2020
2 parents 2cca3e5 + 0c9ce16 commit 3e5367e
Show file tree
Hide file tree
Showing 13 changed files with 757 additions and 94 deletions.
360 changes: 339 additions & 21 deletions LICENSE
View file

Large diffs are not rendered by default.

134 changes: 122 additions & 12 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,25 @@ Framework to train semantic segmentation models on TensorFlow using json files a

```
{
"name": "Example",
"epochs": 100,
"name": "test",
"epochs": 4,
"experiment_data_path": "/data/test",
"checkpoint_frequency": 1,
"checkpoint_frequency": 10,
"warmup_epochs": 2,
"use_multiple_gpus": false,
"hyperparameters": {
"batch_size": 16,
"optimizer": {
"name": "Adam",
"config": {
"learning_rate": 0.01
"learning_rate": 0.0001
}
}
},
"train_dataset": {
"name": "train_ds",
"file_path": "/data/dataset_test.csv",
"file_path": "/data/train_ds.csv",
"n_classes": 1,
"dataset_size": 1000,
"dataset_size": 60000,
"augmentation_list": [
{
"name": "random_crop",
Expand All @@ -50,18 +48,16 @@ Framework to train semantic segmentation models on TensorFlow using json files a
"num_paralel_reads": 4,
"img_dtype": "float32",
"img_format": "png",
"img_width": 256,
"img_length": 256,
"img_width": 512,
"img_length": 512,
"use_ds_width_len": false,
"autotune": -1,
"distributed_training": false
},
"test_dataset": {
"name": "test_ds",
"file_path": "/data/dataset_test.csv",
"file_path": "/data/test_ds.csv",
"n_classes": 1,
"dataset_size": 1000,
"dataset_size": 40000,
"augmentation_list": [
{
"name": "random_crop",
Expand All @@ -70,6 +66,40 @@ Framework to train semantic segmentation models on TensorFlow using json files a
"crop_height": 256
}
},
{
"name": "random_flip_left_right",
"parameters": {}
},
{
"name": "random_flip_up_down",
"parameters": {}
},
{
"name": "random_brightness",
"parameters": {
"max_delta": 0.1
}
},
{
"name": "random_contrast",
"parameters": {
"lower": 0.5,
"upper": 1.5
}
},
{
"name": "random_saturation",
"parameters": {
"lower": 0.5,
"upper": 1.5
}
},
{
"name": "random_hue",
"parameters": {
"max_delta": 0.01
}
},
{
"name": "per_image_standardization",
"parameters": {}
Expand All @@ -95,6 +125,86 @@ Framework to train semantic segmentation models on TensorFlow using json files a
"architecture": "Unet",
"activation": "sigmoid",
"use_imagenet_weights": true
},
"loss": {
"class_name": "bce_dice_loss",
"config": {},
"framework": "sm"
},
"callbacks": {
"items": [
{
"name": "TensorBoard",
"config": {
"update_freq": "epoch"
}
},
{
"name": "BackupAndRestore",
"config": {}
},
{
"name": "ReduceLROnPlateau",
"config": {
"monitor": "val_loss",
"factor": 0.2,
"patience": 5,
"min_lr": 0.00000000001
}
},
{
"name": "ModelCheckpoint",
"config": {
"monitor": "iou_score",
"save_best_only": false,
"save_weights_only": false,
"verbose":1
}
},
{
"name": "ImageHistory",
"config": {
"draw_interval": 1,
"page_size": 10
}
}
]
},
"metrics": {
"items": [
{
"class_name": "iou_score",
"config": {},
"framework": "sm"
},
{
"class_name": "precision",
"config": {},
"framework": "sm"
},
{
"class_name": "recall",
"config": {},
"framework": "sm"
},
{
"class_name": "f1_score",
"config": {},
"framework": "sm"
},
{
"class_name": "f2_score",
"config": {},
"framework": "sm"
},
{
"class_name": "MeanIoU",
"config": {
"num_classes": 2
},
"framework": "tf.keras"
}
]
}
}
```
Expand Down
5 changes: 2 additions & 3 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
tensorflow
git+https://github.com/phborba/efficientnet
git+https://github.com/phborba/segmentation_models
tensorflow>=2.3.0
segmentation_models
pillow
matplotlib
numpy
Expand Down
18 changes: 14 additions & 4 deletions segmentation_models_trainer/callbacks_loader/callback.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@ class Callback(JsonSchemaMixin):
name: str
config: dict

def __post_init__(self):
self.callback_obj = self.get_callback()

@staticmethod
def validate_callback_name(name):
if name not in [
Expand Down Expand Up @@ -93,4 +90,17 @@ def get_tf_objects(self):
]
y
z = CallbackList(y)
print(z.to_json())
print(z.to_json())
w = Callback(
'ImageHistory',
config={
'tensor_board_dir' : '/teste',
'data' : None,
'n_epochs' : 4,
'draw_interval' : 1,
'batch_size' : 1,
'page_size' : 1,
'report_dir': '/teste'
}
)
w
14 changes: 14 additions & 0 deletions segmentation_models_trainer/callbacks_loader/callback_factory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,17 @@ def get_callback(name, parameters):
return tf.keras.callbacks.TensorBoard(
**parameters
)

if __name__ == '__main__':
img_callback = CallbackFactory.get_callback(
'ImageHistory',
parameters={
'tensor_board_dir' : '/teste',
'data' : None,
'n_epochs' : 4,
'draw_interval' : 1,
'batch_size' : 1,
'page_size' : 1,
'report_dir': '/teste'
}
)
34 changes: 22 additions & 12 deletions segmentation_models_trainer/callbacks_loader/custom_callbacks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,29 @@ class ImageHistory(tf.keras.callbacks.Callback):

def __init__(self, params, **kwargs):
super(ImageHistory, self).__init__(**kwargs)
self.tensor_board_dir = params['tensor_board_dir']
self.data = params['data']
self.last_epoch = 0 if 'current_epoch' not in params \
else params['current_epoch']
self.n_epochs = params['n_epochs']
self.draw_interval = params['draw_interval']
self.batch_size = params['batch_size']
self.page_size = params['page_size']
self.report_dir = params['report_dir']
self.tensorboard_dir = params['tensorboard_dir'] if 'tensorboard_dir' in params else None
self.dataset = params['dataset'] if 'dataset' in params else None
self.n_epochs = params['n_epochs'] if 'n_epochs' in params else 1
self.draw_interval = params['draw_interval'] if 'draw_interval' in params else 1
self.batch_size = params['batch_size'] if 'batch_size' in params else 1
self.page_size = params['page_size'] if 'page_size' in params else self.batch_size
self.report_dir = params['report_dir'] if 'report_dir' in params else None

def set_params(self, params):
if 'dataset' in params:
self.dataset = params['dataset']
if 'tensorboard_dir' in params:
self.tensorboard_dir = params['tensorboard_dir']
if 'n_epochs' in params:
self.n_epochs = params['n_epochs']
if 'batch_size' in params:
self.batch_size = params['batch_size']
if 'report_dir' in params:
self.report_dir = params['report_dir']

def on_epoch_end(self, epoch, logs=None):
self.my_logs = logs or {}
self.last_epoch += 1
self.last_epoch = epoch

image_data, label_data, y_pred, data = self.predict_data()
args = [
Expand All @@ -57,7 +67,7 @@ def on_epoch_end(self, epoch, logs=None):
] * self.page_size

file_writer = tf.summary.create_file_writer(
self.tensor_board_dir
self.tensorboard_dir
)
n_pages = np.ceil(
self.batch_size / self.page_size
Expand Down Expand Up @@ -108,7 +118,7 @@ def predict_data(self):
predicted_images = []
ref_labels = []
image_data, label_data = list(
self.data.take(1)
self.dataset.take(1)
)[0]
#took one batch
y_pred = self.model.predict(image_data)
Expand Down
39 changes: 30 additions & 9 deletions segmentation_models_trainer/dataset_loader/dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@
* *
****
"""

import tensorflow as tf
import os
from dataclasses import dataclass, field
from dataclasses_jsonschema import JsonSchemaMixin
from typing import Any, List
from collections import OrderedDict
import tensorflow as tf


IMAGE_DTYPE = {
'float32' : tf.float32,
Expand Down Expand Up @@ -121,10 +122,11 @@ def augment_image(self, input_image, input_mask):
@dataclass
class Dataset(JsonSchemaMixin):
name: str
file_path: str
n_classes: int
dataset_size: int
augmentation_list: List[ImageAugumentation]
file_path: str
base_path: str = ''
cache: Any = True
shuffle: bool = True
shuffle_buffer_size: int = 10000
Expand All @@ -133,8 +135,8 @@ class Dataset(JsonSchemaMixin):
num_paralel_reads: int = 4
img_dtype: str = 'float32'
img_format: str = 'png'
img_width: int = 256
img_length: int = 256
img_width: int = 512
img_length: int = 512
img_bands: int = 3
mask_bands: int = 1
use_ds_width_len: bool = False
Expand All @@ -149,16 +151,31 @@ def get_img_input_shape(self):
)

def get_tf_dataset(self, batch_size):
@tf.function
def process_csv_entry(entry):
width = entry['width'] if self.use_ds_width_len else self.img_width
length = entry['length'] if self.use_ds_width_len else self.img_length
label = tf.io.read_file(
entry['label_path'][0]
entry['label_path'][0] if self.base_path == '' \
else tf.strings.join(
[
self.base_path,
entry['label_path'][0]
],
separator=''
)
)
label = decode_img(label, width, length, channels=1)
# load the raw data from the file as a string
img = tf.io.read_file(
entry['image_path'][0]
entry['label_path'][0] if self.base_path == '' \
else tf.strings.join(
[
self.base_path,
entry['label_path'][0]
],
separator=''
)
)
img = decode_img(img, width, length)
img, label = augment_image(img, label)
Expand Down Expand Up @@ -186,7 +203,10 @@ def prepare_for_training(ds, batch_size):
ds = ds.repeat()
# `prefetch` lets the dataset fetch batches in the background while the model
# is training.
ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE if self.autotune == -1 else self.autotune)
ds = ds.prefetch(
buffer_size=tf.data.experimental.AUTOTUNE \
if self.autotune == -1 else self.autotune
)
return ds

def augment_image(img, label):
Expand All @@ -205,7 +225,8 @@ def augment_image(img, label):
)
labeled_ds = ds.map(
process_csv_entry,
num_parallel_calls=tf.data.experimental.AUTOTUNE if self.autotune == -1 else self.autotune
num_parallel_calls=tf.data.experimental.AUTOTUNE if self.autotune == -1 \
else self.autotune
)
prepared_ds = prepare_for_training(
labeled_ds,
Expand Down
Loading

0 comments on commit 3e5367e

Please sign in to comment.