diff --git a/.github/workflows/nn-ci-cpu-testing.yml b/.github/workflows/nn-ci-cpu-testing.yml index 53274a34..73828e11 100644 --- a/.github/workflows/nn-ci-cpu-testing.yml +++ b/.github/workflows/nn-ci-cpu-testing.yml @@ -5,11 +5,11 @@ name: Nomeroff Net CI CPU Testing on: push: - branches: [ v3.0, master ] + branches: [ master v3.0 v3.1 ] pull_request: - branches: [ v3.0, master ] - schedule: - - cron: '0 0 * * *' # Runs at 00:00 UTC every day + branches: [ master v3.0 v3.1 ] +# schedule: +# - cron: '0 0 * * *' # Runs at 00:00 UTC every day jobs: cpu-tests: diff --git a/History.md b/History.md index 140c3cd1..5f2e5645 100644 --- a/History.md +++ b/History.md @@ -1,9 +1,16 @@ -3.0.0 / 2021-11-24 +3.1.0 / 2022-03-28 +================== + **updates** + * Returned to a separate backbone for ocr models + * Fixed bag with block_cnn in ocr models + +3.0.0 / 2022-03-16 ================== **updates** * Refactored code with Sonarqube * Added Pipelines * Restructured code + * Added common backbone for ocr models 2.5.0 / 2021-11-24 ================== diff --git a/README.md b/README.md index 94055216..711f2fe8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![Nomeroff Net. Automatic numberplate recognition system](./public/images/nomeroff_net.svg) -Nomeroff Net. Automatic numberplate recognition system. Version 3.0 +Nomeroff Net. Automatic numberplate recognition system. Version 3.1

Now there is a war going on in my country, Russian soldiers are shooting at civilians in Ukraine. Enemy aviation launches rockets and drops bombs on residential quarters. diff --git a/nomeroff_net/__init__.py b/nomeroff_net/__init__.py index ac45bc49..f1302930 100644 --- a/nomeroff_net/__init__.py +++ b/nomeroff_net/__init__.py @@ -8,4 +8,4 @@ from nomeroff_net.pipelines import pipeline -__version__ = "3.0.0" +__version__ = "3.1.0" diff --git a/nomeroff_net/data_loaders/text_image_generator.py b/nomeroff_net/data_loaders/text_image_generator.py index a48c9fb5..59fa94d2 100644 --- a/nomeroff_net/data_loaders/text_image_generator.py +++ b/nomeroff_net/data_loaders/text_image_generator.py @@ -3,14 +3,12 @@ import json import torch import numpy as np -import torch.nn as nn from tqdm import tqdm from PIL import Image from typing import List, Tuple, Generator, Any from torchvision import transforms -from torchvision.models import resnet18 -from nomeroff_net.tools.mcm import modelhub, get_device_torch +from nomeroff_net.tools.mcm import get_device_torch from nomeroff_net.tools.ocr_tools import is_valid_str device_torch = get_device_torch() @@ -26,6 +24,7 @@ def __init__(self, img_h: int = 64, batch_size: int = 1, max_plate_length: int = 8, + seed: int = 42, with_aug: bool = False) -> None: self.dirpath = dirpath @@ -43,7 +42,7 @@ def __init__(self, ann_dirpath = os.path.join(dirpath, 'ann') cache_postfix = "cache_ocr" if with_aug: - cache_postfix = f"{cache_postfix}_aug" + cache_postfix = f"{cache_postfix}_aug_{seed}" cache_dirpath = os.path.join(dirpath, cache_postfix) os.makedirs(cache_dirpath, exist_ok=True) self.pathes = [os.path.join(img_dirpath, file_name) for file_name in os.listdir(img_dirpath)] @@ -116,15 +115,7 @@ def __getitem__(self, index): img = self.get_x_from_path(img_path) return img, text - def prepare_transformers(self, model_name="Resnet18"): - model_info = modelhub.download_model_by_name(model_name) - path_to_model = model_info["path"] - - resnet = resnet18(pretrained=False) - modules = list(resnet.children())[:-3] - self.resnet = nn.Sequential(*modules) - self.resnet.load_state_dict(torch.load(path_to_model, map_location=device_torch)) - self.resnet = self.resnet.to(device_torch) + def prepare_transformers(self): self.list_transforms = transforms.Compose([ transforms.ToTensor(), ]) @@ -132,9 +123,6 @@ def prepare_transformers(self, model_name="Resnet18"): @torch.no_grad() def transform(self, img) -> torch.Tensor: x = self.list_transforms(img) - x = x.unsqueeze(0).to(device_torch) - x = self.resnet(x) - x = x.squeeze(0).cpu() return x def next_sample(self) -> Tuple: diff --git a/nomeroff_net/data_modules/numberplate_ocr_data_module.py b/nomeroff_net/data_modules/numberplate_ocr_data_module.py index 2b635690..16a58284 100644 --- a/nomeroff_net/data_modules/numberplate_ocr_data_module.py +++ b/nomeroff_net/data_modules/numberplate_ocr_data_module.py @@ -18,6 +18,7 @@ def __init__(self, batch_size=32, max_plate_length=8, num_workers=0, + seed=42, with_aug=False): super().__init__() self.batch_size = batch_size @@ -33,6 +34,7 @@ def __init__(self, img_h=height, batch_size=batch_size, max_plate_length=max_plate_length, + seed=seed, with_aug=with_aug) # init validation generator diff --git a/nomeroff_net/nnmodels/ocr_model.py b/nomeroff_net/nnmodels/ocr_model.py index dfb0be09..4835f041 100644 --- a/nomeroff_net/nnmodels/ocr_model.py +++ b/nomeroff_net/nnmodels/ocr_model.py @@ -4,10 +4,10 @@ """ import torch import torch.nn as nn +from typing import List, Any import pytorch_lightning as pl - from torch.nn import functional -from typing import List, Any +from torchvision.models import resnet18 from nomeroff_net.tools.ocr_tools import plot_loss, print_prediction @@ -21,39 +21,6 @@ def weights_init(m): m.bias.data.fill_(0) -class BlockCNN(nn.Module): - def __init__(self, in_nc, out_nc, kernel_size, padding, stride=tuple([1])): - super(BlockCNN, self).__init__() - self.in_nc = in_nc - self.out_nc = out_nc - self.kernel_size = kernel_size - self.padding = padding - # layers - self.conv = nn.Conv2d(in_nc, out_nc, - kernel_size=kernel_size, - stride=stride, - padding=padding) - self.bn = nn.BatchNorm2d(out_nc) - - def forward(self, batch, use_bn=False, use_relu=False, - use_maxpool=False, maxpool_kernelsize=None): - """ - in: - batch - [batch_size, in_nc, H, W] - out: - batch - [batch_size, out_nc, H', W'] - """ - batch = self.conv(batch) - if use_bn: - batch = self.bn(batch) - if use_relu: - batch = functional.relu(batch) - if use_maxpool: - assert maxpool_kernelsize is not None - batch = functional.max_pool2d(batch, kernel_size=maxpool_kernelsize, stride=1) - return batch - - class BlockRNN(nn.Module): def __init__(self, in_size, hidden_size, out_size, bidirectional): super(BlockRNN, self).__init__() @@ -105,8 +72,11 @@ def __init__(self, self.bidirectional = bidirectional self.label_converter = label_converter - - self.cnn = BlockCNN(256, 256, kernel_size=3, padding=1) + + # convolutions + resnet = resnet18(pretrained=True) + modules = list(resnet.children())[:-3] + self.resnet = nn.Sequential(*modules) # RNN + Linear self.linear1 = nn.Linear(1024, 512) @@ -135,6 +105,9 @@ def forward(self, batch: torch.float64): torch.Size([32, batch_size, vocab_size]) -- :OUT """ batch_size = batch.size(0) + + # convolutions + batch = self.resnet(batch) # make sequences of image features batch = batch.permute(0, 3, 1, 2) @@ -195,26 +168,43 @@ def configure_optimizers(self): nesterov=True, weight_decay=self.weight_decay, momentum=self.momentum) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, verbose=True, patience=5) - lr_schedulers = {'scheduler': scheduler, 'monitor': 'val_loss'} - return [optimizer], [lr_schedulers] + return optimizer def training_step(self, batch, batch_idx): loss = self.step(batch) - self.log(f'train_loss', loss) - self.train_losses.append(loss.cpu().detach().numpy()) - return loss + self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + tqdm_dict = { + 'train_loss': loss, + } + return { + 'loss': loss, + 'progress_bar': tqdm_dict, + 'log': tqdm_dict + } def validation_step(self, batch, batch_idx): loss = self.step(batch) - self.log('val_loss', loss) - self.val_losses.append(loss.cpu().detach().numpy()) - return loss + self.log('val_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True) + tqdm_dict = { + 'val_loss': loss, + } + return { + 'val_loss': loss, + 'progress_bar': tqdm_dict, + 'log': tqdm_dict + } def test_step(self, batch, batch_idx): loss = self.step(batch) - self.log('test_loss', loss) - return loss + self.log('test_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True) + tqdm_dict = { + 'test_loss': loss, + } + return { + 'test_loss': loss, + 'progress_bar': tqdm_dict, + 'log': tqdm_dict + } if __name__ == "__main__": diff --git a/nomeroff_net/pipes/number_plate_text_readers/text_detector.py b/nomeroff_net/pipes/number_plate_text_readers/text_detector.py index b15324b6..34833f05 100644 --- a/nomeroff_net/pipes/number_plate_text_readers/text_detector.py +++ b/nomeroff_net/pipes/number_plate_text_readers/text_detector.py @@ -4,7 +4,6 @@ from torch import no_grad from nomeroff_net import text_detectors -from nomeroff_net.pipes.base.resnet18 import Resnet18 from nomeroff_net.tools.errors import TextDetectorError from nomeroff_net.tools.image_processing import convert_cv_zones_rgb_to_bgr @@ -43,7 +42,6 @@ def __init__(self, self.detectors_names.append(_label) i += 1 - self.resnet18 = Resnet18() if load_models: self.load() @@ -51,7 +49,6 @@ def load(self): """ TODO: support reloading """ - self.resnet18.load() for i, (detector_class, detector_name) in enumerate(zip(self.detectors, self.detectors_names)): detector = detector_class() detector.load(self.prisets[detector_name]['model_path']) @@ -100,13 +97,13 @@ def preprocess(self, labels, lines = self.define_predict_classes(zones, labels, lines) predicted = self.define_order_detector(zones, labels) for key in predicted.keys(): - predicted[key]["xs"] = self.resnet18.preprocess(predicted[key]["zones"]) + predicted[key]["xs"] = self.detectors[int(key)].preprocess(predicted[key]["zones"]) return predicted @no_grad() def forward(self, predicted): for key in predicted.keys(): - xs = self.resnet18.forward(predicted[key]["xs"]) + xs = predicted[key]["xs"] predicted[key]["ys"] = self.detectors[int(key)].forward(xs) return predicted diff --git a/nomeroff_net/text_detectors/base/ocr.py b/nomeroff_net/text_detectors/base/ocr.py index aea1057b..4e1ceca3 100644 --- a/nomeroff_net/text_detectors/base/ocr.py +++ b/nomeroff_net/text_detectors/base/ocr.py @@ -13,6 +13,7 @@ from nomeroff_net.data_modules.numberplate_ocr_data_module import OcrNetDataModule from nomeroff_net.nnmodels.ocr_model import NPOcrNet, weights_init +from nomeroff_net.tools.image_processing import normalize_img from nomeroff_net.tools.errors import OCRError from nomeroff_net.tools.mcm import modelhub, get_device_torch from nomeroff_net.tools.augmentations import aug_seed @@ -104,6 +105,7 @@ def get_alphabet(self, train_path: str, test_path: str, val_path: str, verbose: def prepare(self, path_to_dataset: str, use_aug: bool = False, + seed: int = 42, verbose: bool = True, num_workers: int = 0) -> None: train_dir = os.path.join(path_to_dataset, "train") @@ -133,6 +135,7 @@ def prepare(self, batch_size=self.batch_size, max_plate_length=self.max_plate_length, num_workers=num_workers, + seed=seed, with_aug=use_aug) if verbose: print("DATA PREPARED") @@ -150,8 +153,9 @@ def create_model(self) -> NPOcrNet: return self.model def train(self, - log_dir=os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/logs/ocr')), - seed: int = None + log_dir=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../data/logs/ocr')), + seed: int = None, + ckpt_path: str = None ) -> NPOcrNet: """ TODO: describe method @@ -165,9 +169,8 @@ def train(self, lr_monitor = LearningRateMonitor(logging_interval='step') self.trainer = pl.Trainer(max_epochs=self.epochs, gpus=self.gpus, - callbacks=[checkpoint_callback, lr_monitor], - weights_summary=None) - self.trainer.fit(self.model, self.dm) + callbacks=[checkpoint_callback, lr_monitor]) + self.trainer.fit(self.model, self.dm, ckpt_path=ckpt_path) print("[INFO] best model path", checkpoint_callback.best_model_path) return self.model @@ -180,29 +183,38 @@ def validation(self, val_losses, device): val_losses.append(val_loss.item()) return val_losses - def tune(self, percentage=0.1) -> Dict: + def tune(self, percentage=0.05) -> Dict: """ TODO: describe method """ - model = self.create_model() + if self.model is None: + self.create_model() trainer = pl.Trainer(auto_lr_find=True, max_epochs=self.epochs, gpus=self.gpus) num_training = int(len(self.dm.train_image_generator)*percentage) or 1 - lr_finder = trainer.tuner.lr_find(model, + lr_finder = trainer.tuner.lr_find(self.model, self.dm, num_training=num_training, early_stop_threshold=None) lr = lr_finder.suggestion() print(f"Found lr: {lr}") - model.hparams["learning_rate"] = lr + self.model.hparams["learning_rate"] = lr return lr_finder - @staticmethod - def preprocess(xs): + def preprocess(self, imgs): + xs = [] + for img in imgs: + x = normalize_img(img, + width=self.width, + height=self.height) + xs.append(x) + xs = np.moveaxis(np.array(xs), 3, 1) + xs = torch.tensor(xs) + xs = xs.to(device_torch) return xs def forward(self, xs): @@ -250,9 +262,9 @@ def is_loaded(self) -> bool: return False return True - def load_model(self, path_to_model): + def load_model(self, path_to_model, nn_class=NPOcrNet): self.path_to_model = path_to_model - self.model = NPOcrNet.load_from_checkpoint(path_to_model, + self.model = nn_class.load_from_checkpoint(path_to_model, map_location=torch.device('cpu'), letters=self.letters, letters_max=len(self.letters) + 1, @@ -262,7 +274,7 @@ def load_model(self, path_to_model): self.model.eval() return self.model - def load(self, path_to_model: str = "latest") -> NPOcrNet: + def load(self, path_to_model: str = "latest", nn_class=NPOcrNet) -> NPOcrNet: """ TODO: describe method """ @@ -276,7 +288,7 @@ def load(self, path_to_model: str = "latest") -> NPOcrNet: self.get_classname()) path_to_model = model_info["path"] - return self.load_model(path_to_model) + return self.load_model(path_to_model, nn_class=nn_class) @torch.no_grad() def get_acc(self, predicted: List, decode: List) -> torch.Tensor: @@ -303,21 +315,22 @@ def get_acc(self, predicted: List, decode: List) -> torch.Tensor: text_lens ) return 1 - acc / len(self.letters) - + + @torch.no_grad() def acc_calc(self, dataset, verbose: bool = False) -> float: acc = 0 - with torch.no_grad(): - self.model.eval() - for idx in range(len(dataset)): - img, text = dataset[idx] - img = img.unsqueeze(0).to(device_torch) - logits = self.model(img) - pred_text = decode_prediction(logits.cpu(), self.label_converter) - - if pred_text == text: - acc += 1 - elif verbose: - print(f'\n[INFO] {dataset.pathes[idx]}\nPredicted: {pred_text} \t\t\t True: {text}') + self.model = self.model.to(device_torch) + self.model.eval() + for idx in range(len(dataset)): + img, text = dataset[idx] + img = img.unsqueeze(0).to(device_torch) + logits = self.model(img) + pred_text = decode_prediction(logits.cpu(), self.label_converter) + + if pred_text == text: + acc += 1 + elif verbose: + print(f'\n[INFO] {dataset.pathes[idx]}\nPredicted: {pred_text} \t\t\t True: {text}') return acc / len(dataset) def val_acc(self, verbose=False) -> float: diff --git a/nomeroff_net/text_detectors/base/ocr_trt.py b/nomeroff_net/text_detectors/base/ocr_trt.py index 393f9c6c..e69de29b 100644 --- a/nomeroff_net/text_detectors/base/ocr_trt.py +++ b/nomeroff_net/text_detectors/base/ocr_trt.py @@ -1,69 +0,0 @@ -from typing import List, Any, Dict - -import onnxruntime -import numpy as np -import torch - -from nomeroff_net.tools import modelhub -from nomeroff_net.tools.image_processing import normalize_img -from nomeroff_net.tools.ocr_tools import decode_batch -from .ocr import OCR - - -class OcrTrt(OCR): - def __init__(self) -> None: - OCR.__init__(self) - self.ort_session = None - self.input_name = None - - def is_loaded(self) -> bool: - if self.ort_session is None: - return False - return True - - def load_model(self, path_to_model): - self.ort_session = onnxruntime.InferenceSession(path_to_model) - self.input_name = self.ort_session.get_inputs()[0].name - return self.ort_session - - def load(self, path_to_model: str = "latest", options: Dict = None) -> onnxruntime.InferenceSession: - """ - TODO: describe method - """ - if options is None: - options = dict() - self.__dict__.update(options) - if path_to_model == "latest": - model_info = modelhub.download_model_by_name(self.get_classname()) - path_to_model = model_info["path"] - elif path_to_model.startswith("http"): - model_info = modelhub.download_model_by_url(path_to_model, - self.get_classname(), - self.get_classname()) - path_to_model = model_info["path"] - return self.load_model(path_to_model) - - @torch.no_grad() - def predict(self, imgs: List, return_acc: bool = False) -> Any: - xs = [] - for img in imgs: - x = normalize_img(img, - width=self.width, - height=self.height) - xs.append(x) - pred_texts = [] - net_out_value = [] - if bool(xs): - xs = np.moveaxis(np.array(xs), 3, 1) - ort_inputs = {self.input_name: xs} - net_out_value = self.ort_session.run(None, ort_inputs)[0] - pred_texts = decode_batch(torch.Tensor(net_out_value), self.label_converter) - pred_texts = [pred_text.upper() for pred_text in pred_texts] - if return_acc: - if len(net_out_value): - net_out_value = np.array(net_out_value) - net_out_value = net_out_value.reshape((net_out_value.shape[1], - net_out_value.shape[0], - net_out_value.shape[2])) - return pred_texts, net_out_value - return pred_texts diff --git a/nomeroff_net/tools/mcm.py b/nomeroff_net/tools/mcm.py index 5bf04d36..1bc052d5 100644 --- a/nomeroff_net/tools/mcm.py +++ b/nomeroff_net/tools/mcm.py @@ -4,18 +4,17 @@ model_config_urls = [ "http://models.vsp.net.ua/config_model/nomeroff-net-np-classification/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-am/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-base/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-by/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu_ua_1995/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu_ua_from_2004/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ge/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-kg/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-kz/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ru/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ru-military/model-1.json", - "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-su/model-1.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-am/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-by/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu_ua_1995/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-eu_ua_from_2004/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ge/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-kg/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-kz/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ru/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-ru-military/model-2.json", + "http://models.vsp.net.ua/config_model/nomeroff-net-ocr-su/model-2.json", "http://models.vsp.net.ua/config_model/nomeroff-net-ua-np-classification/model-1.json", "http://models.vsp.net.ua/config_model/nomeroff-net-yolov5/model-1.json", "http://models.vsp.net.ua/config_model/craft-mlt/model-1.json",