From 242cac7d8405769b69af8cb82c3b8720da98471c Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Thu, 11 Apr 2024 17:57:14 +0200 Subject: [PATCH 01/12] feat: cleanup code for the release. --- README.md | 2 +- docs/source/user_guide/faster_rcnn_example.py | 32 +++++----- docs/source/user_guide/getting_started.ipynb | 2 +- tests/config/common/__init__.py | 1 - tests/config/registry_test.py | 4 +- tests/config/show_connection_test.py | 11 +--- tests/config/sweep/__init__.py | 1 - tests/config/{sweep => }/sweep_test.py | 4 +- tests/config/testcases/connection.txt | 62 +++++++++++++++++++ tests/engine/callbacks/evaluator_test.py | 2 +- tests/engine/callbacks/visualizer_test.py | 2 +- tests/engine/optim/optimizer_test.py | 4 +- tests/model/detect/mask_rcnn_test.py | 10 +-- tests/pl/data_module_test.py | 2 +- tests/pl/trainer_test.py | 2 +- tests/util.py | 40 ++++++++++++ tests/zoo/base/__init__.py | 1 + .../common => zoo/base}/datasets/__init__.py | 0 .../base}/datasets/bdd100k/__init__.py | 0 .../base}/datasets/bdd100k/detect_test.py | 4 +- .../base}/datasets/bdd100k/sem_seg_test.py | 4 +- .../base}/datasets/coco/__init__.py | 0 .../base}/datasets/coco/detection_test.py | 2 +- .../base}/datasets/nuscenes/__init__.py | 0 .../datasets/nuscenes/nuscenes_mono_test.py | 2 +- .../base}/datasets/nuscenes/nuscenes_test.py | 4 +- .../base}/datasets/shift/__init__.py | 0 .../base}/datasets/shift/test_shift_tasks.py | 6 +- tests/zoo/bdd100k/faster_rcnn_test.py | 2 +- tests/zoo/bdd100k/mask_rcnn_test.py | 2 +- tests/zoo/bdd100k/qdtrack_test.py | 2 +- tests/zoo/bdd100k/semantic_fpn_test.py | 2 +- tests/zoo/bevformer_test.py | 2 +- tests/zoo/cc_3dt_test.py | 2 +- tests/zoo/faster_rcnn_test.py | 2 +- tests/zoo/fcn_resnet_test.py | 2 +- tests/zoo/mask_rcnn_test.py | 2 +- tests/zoo/qdtrack_test.py | 2 +- tests/zoo/retinanet_test.py | 2 +- tests/zoo/shift/faster_rcnn_test.py | 2 +- tests/zoo/shift/mask_rcnn_test.py | 2 +- tests/zoo/shift/semantic_fpn_test.py | 2 +- tests/zoo/util.py | 43 +------------ tests/zoo/vit_imagenet_test.py | 2 +- tests/zoo/yolox_test.py | 2 +- vis4d/common/typing.py | 2 + vis4d/common/util.py | 43 +++++++++---- vis4d/config/common/__init__.py | 1 - vis4d/config/common/datasets/__init__.py | 1 - .../common/datasets/imagenet/__init__.py | 5 -- vis4d/config/common/models/__init__.py | 6 -- vis4d/config/default/__init__.py | 10 --- vis4d/config/{util => }/registry.py | 2 +- vis4d/config/{util => }/sweep.py | 0 vis4d/engine/experiment.py | 5 +- vis4d/engine/optim/optimizer.py | 2 +- vis4d/engine/optim/scheduler.py | 2 +- vis4d/engine/parser.py | 2 +- vis4d/engine/run.py | 2 +- vis4d/pl/data_module.py | 2 +- vis4d/pl/run.py | 6 +- vis4d/pl/training_module.py | 2 +- vis4d/state/track/qdtrack.py | 2 +- vis4d/{config/util => zoo/base}/__init__.py | 7 ++- vis4d/{config/util => zoo/base}/callable.py | 0 .../base}/data_connectors/__init__.py | 2 +- .../base}/data_connectors/cls.py | 2 +- .../base}/data_connectors/common.py | 2 +- .../base}/data_connectors/detection.py | 2 +- .../base}/data_connectors/seg.py | 2 +- .../base}/data_connectors/visualizers.py | 0 vis4d/{config/util => zoo/base}/dataloader.py | 0 vis4d/zoo/base/datasets/__init__.py | 1 + .../base}/datasets/bdd100k/__init__.py | 0 .../base}/datasets/bdd100k/detect.py | 10 +-- .../base}/datasets/bdd100k/sem_seg.py | 10 +-- .../base}/datasets/bdd100k/track.py | 10 +-- .../base}/datasets/coco/__init__.py | 0 .../base}/datasets/coco/detection.py | 10 +-- .../base}/datasets/coco/sem_seg.py | 10 +-- .../cls.py => zoo/base/datasets/imagenet.py} | 4 +- .../base}/datasets/nuscenes/__init__.py | 0 .../base}/datasets/nuscenes/nuscenes.py | 0 .../base}/datasets/nuscenes/nuscenes_mono.py | 0 .../base}/datasets/shift/__init__.py | 0 .../base}/datasets/shift/common.py | 10 ++- .../base}/datasets/shift/tasks.py | 0 vis4d/zoo/base/models/__init__.py | 1 + .../common => zoo/base}/models/faster_rcnn.py | 0 .../common => zoo/base}/models/mask_rcnn.py | 13 ++-- .../common => zoo/base}/models/qdtrack.py | 12 ++-- .../common => zoo/base}/models/yolox.py | 4 +- vis4d/{config/util => zoo/base}/optimizer.py | 6 +- .../default => zoo/base}/pl_trainer.py | 2 +- vis4d/{config/default => zoo/base}/runtime.py | 3 +- .../faster_rcnn/faster_rcnn_r50_1x_bdd100k.py | 31 +++++----- .../faster_rcnn/faster_rcnn_r50_3x_bdd100k.py | 31 +++++----- .../mask_rcnn/mask_rcnn_r50_1x_bdd100k.py | 33 +++++----- .../mask_rcnn/mask_rcnn_r50_3x_bdd100k.py | 33 +++++----- .../mask_rcnn/mask_rcnn_r50_5x_bdd100k.py | 33 +++++----- .../qdtrack_frcnn_r50_fpn_1x_bdd100k.py | 31 +++++----- .../semantic_fpn_r101_80k_bdd100k.py | 35 ++++++----- .../semantic_fpn_r50_40k_bdd100k.py | 35 ++++++----- .../semantic_fpn_r50_80k_bdd100k.py | 35 ++++++----- vis4d/zoo/bevformer/bevformer_base.py | 15 ++--- vis4d/zoo/bevformer/bevformer_tiny.py | 17 ++--- vis4d/zoo/bevformer/bevformer_vis.py | 4 +- vis4d/zoo/bevformer/data.py | 12 ++-- .../cc_3dt_bevformer_base_velo_lstm_nusc.py | 2 +- .../cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py | 15 ++--- .../cc_3dt_frcnn_r101_fpn_pure_det_nusc.py | 4 +- ...c_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py | 2 +- .../cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py | 15 ++--- vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py | 4 +- vis4d/zoo/cc_3dt/data.py | 24 +++---- vis4d/zoo/cc_3dt/model.py | 20 +++--- .../velo_lstm_bevformer_base_100e_nusc.py | 24 +++---- .../velo_lstm_frcnn_r101_fpn_100e_nusc.py | 24 +++---- vis4d/zoo/faster_rcnn/faster_rcnn_coco.py | 33 +++++----- vis4d/zoo/fcn_resnet/fcn_resnet_coco.py | 23 +++---- vis4d/zoo/mask_rcnn/mask_rcnn_coco.py | 35 ++++++----- vis4d/zoo/qdtrack/data_yolox.py | 6 +- .../qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py | 29 ++++----- .../qdtrack/qdtrack_yolox_x_50e_bdd100k.py | 32 +++++----- vis4d/zoo/retinanet/retinanet_coco.py | 35 ++++++----- .../faster_rcnn/faster_rcnn_r50_12e_shift.py | 31 +++++----- .../faster_rcnn/faster_rcnn_r50_36e_shift.py | 31 +++++----- .../faster_rcnn_r50_6e_shift_all_domains.py | 31 +++++----- .../mask_rcnn/mask_rcnn_r50_12e_shift.py | 29 ++++----- .../mask_rcnn/mask_rcnn_r50_36e_shift.py | 29 ++++----- .../mask_rcnn_r50_6e_shift_all_domains.py | 29 ++++----- .../semantic_fpn_r50_160k_shift.py | 31 +++++----- ...semantic_fpn_r50_160k_shift_all_domains.py | 31 +++++----- .../semantic_fpn_r50_40k_shift.py | 31 +++++----- .../semantic_fpn_r50_40k_shift_all_domains.py | 31 +++++----- vis4d/{config => zoo}/typing.py | 3 +- vis4d/zoo/vit/vit_small_imagenet.py | 33 +++++----- vis4d/zoo/vit/vit_tiny_imagenet.py | 33 +++++----- vis4d/zoo/yolox/data.py | 6 +- vis4d/zoo/yolox/yolox_s_300e_coco.py | 27 ++++---- vis4d/zoo/yolox/yolox_tiny_300e_coco.py | 27 ++++---- 141 files changed, 839 insertions(+), 763 deletions(-) delete mode 100644 tests/config/common/__init__.py delete mode 100644 tests/config/sweep/__init__.py rename tests/config/{sweep => }/sweep_test.py (97%) create mode 100644 tests/config/testcases/connection.txt create mode 100644 tests/zoo/base/__init__.py rename tests/{config/common => zoo/base}/datasets/__init__.py (100%) rename tests/{config/common => zoo/base}/datasets/bdd100k/__init__.py (100%) rename tests/{config/common => zoo/base}/datasets/bdd100k/detect_test.py (97%) rename tests/{config/common => zoo/base}/datasets/bdd100k/sem_seg_test.py (97%) rename tests/{config/common => zoo/base}/datasets/coco/__init__.py (100%) rename tests/{config/common => zoo/base}/datasets/coco/detection_test.py (97%) rename tests/{config/common => zoo/base}/datasets/nuscenes/__init__.py (100%) rename tests/{config/common => zoo/base}/datasets/nuscenes/nuscenes_mono_test.py (92%) rename tests/{config/common => zoo/base}/datasets/nuscenes/nuscenes_test.py (96%) rename tests/{config/common => zoo/base}/datasets/shift/__init__.py (100%) rename tests/{config/common => zoo/base}/datasets/shift/test_shift_tasks.py (99%) delete mode 100644 vis4d/config/common/__init__.py delete mode 100644 vis4d/config/common/datasets/__init__.py delete mode 100644 vis4d/config/common/datasets/imagenet/__init__.py delete mode 100644 vis4d/config/common/models/__init__.py delete mode 100644 vis4d/config/default/__init__.py rename vis4d/config/{util => }/registry.py (99%) rename vis4d/config/{util => }/sweep.py (100%) rename vis4d/{config/util => zoo/base}/__init__.py (60%) rename vis4d/{config/util => zoo/base}/callable.py (100%) rename vis4d/{config/default => zoo/base}/data_connectors/__init__.py (92%) rename vis4d/{config/default => zoo/base}/data_connectors/cls.py (88%) rename vis4d/{config/default => zoo/base}/data_connectors/common.py (82%) rename vis4d/{config/default => zoo/base}/data_connectors/detection.py (92%) rename vis4d/{config/default => zoo/base}/data_connectors/seg.py (93%) rename vis4d/{config/default => zoo/base}/data_connectors/visualizers.py (100%) rename vis4d/{config/util => zoo/base}/dataloader.py (100%) create mode 100644 vis4d/zoo/base/datasets/__init__.py rename vis4d/{config/common => zoo/base}/datasets/bdd100k/__init__.py (100%) rename vis4d/{config/common => zoo/base}/datasets/bdd100k/detect.py (98%) rename vis4d/{config/common => zoo/base}/datasets/bdd100k/sem_seg.py (98%) rename vis4d/{config/common => zoo/base}/datasets/bdd100k/track.py (98%) rename vis4d/{config/common => zoo/base}/datasets/coco/__init__.py (100%) rename vis4d/{config/common => zoo/base}/datasets/coco/detection.py (98%) rename vis4d/{config/common => zoo/base}/datasets/coco/sem_seg.py (98%) rename vis4d/{config/common/datasets/imagenet/cls.py => zoo/base/datasets/imagenet.py} (98%) rename vis4d/{config/common => zoo/base}/datasets/nuscenes/__init__.py (100%) rename vis4d/{config/common => zoo/base}/datasets/nuscenes/nuscenes.py (100%) rename vis4d/{config/common => zoo/base}/datasets/nuscenes/nuscenes_mono.py (100%) rename vis4d/{config/common => zoo/base}/datasets/shift/__init__.py (100%) rename vis4d/{config/common => zoo/base}/datasets/shift/common.py (99%) rename vis4d/{config/common => zoo/base}/datasets/shift/tasks.py (100%) create mode 100644 vis4d/zoo/base/models/__init__.py rename vis4d/{config/common => zoo/base}/models/faster_rcnn.py (100%) rename vis4d/{config/common => zoo/base}/models/mask_rcnn.py (94%) rename vis4d/{config/common => zoo/base}/models/qdtrack.py (97%) rename vis4d/{config/common => zoo/base}/models/yolox.py (98%) rename vis4d/{config/util => zoo/base}/optimizer.py (96%) rename vis4d/{config/default => zoo/base}/pl_trainer.py (95%) rename vis4d/{config/default => zoo/base}/runtime.py (96%) rename vis4d/{config => zoo}/typing.py (98%) diff --git a/README.md b/README.md index d23821a36..188b7ed7e 100644 --- a/README.md +++ b/README.md @@ -92,4 +92,4 @@ srun --cpus-per-task=4 --gres=gpumem:20G python -m vis4d.pl.run test \ ## Contribute -[Check out our contribution guidelines for this project](docs/source/contribute.rst) +[Check out our contribution guidelines for this project](docs/source/dev_guide/contribute.rst) diff --git a/docs/source/user_guide/faster_rcnn_example.py b/docs/source/user_guide/faster_rcnn_example.py index 04b5c64da..ab5ec25e2 100644 --- a/docs/source/user_guide/faster_rcnn_example.py +++ b/docs/source/user_guide/faster_rcnn_example.py @@ -2,38 +2,40 @@ """Faster RCNN COCO training example.""" from __future__ import annotations +import tempfile + import lightning.pytorch as pl import numpy as np -import tempfile from torch.optim import SGD from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.coco import ( - CONN_COCO_BBOX_EVAL, - get_coco_detection_cfg, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.config.sweep import grid_search +from vis4d.engine.callbacks import EvaluatorCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.coco import COCODetectEvaluator +from vis4d.op.base import ResNet +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, ) -from vis4d.config.typing import ( +from vis4d.zoo.base.datasets.coco import ( + CONN_COCO_BBOX_EVAL, + get_coco_detection_cfg, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ( ExperimentConfig, ExperimentParameters, ParameterSweepConfig, ) -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.config.util.sweep import grid_search -from vis4d.engine.callbacks import EvaluatorCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.coco import COCODetectEvaluator -from vis4d.op.base import ResNet def get_config() -> ExperimentConfig: diff --git a/docs/source/user_guide/getting_started.ipynb b/docs/source/user_guide/getting_started.ipynb index 118c13080..db99ef46e 100644 --- a/docs/source/user_guide/getting_started.ipynb +++ b/docs/source/user_guide/getting_started.ipynb @@ -254,7 +254,7 @@ "from vis4d.vis.functional.image import imshow_bboxes\n", "\n", "from vis4d.config import instantiate_classes\n", - "from vis4d.config.common.datasets.coco import get_coco_detection_cfg" + "from vis4d.zoo.base.datasets.coco import get_coco_detection_cfg" ] }, { diff --git a/tests/config/common/__init__.py b/tests/config/common/__init__.py deleted file mode 100644 index ea403e5fe..000000000 --- a/tests/config/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Common config tests.""" diff --git a/tests/config/registry_test.py b/tests/config/registry_test.py index efe1c5e1c..d054bdc7f 100644 --- a/tests/config/registry_test.py +++ b/tests/config/registry_test.py @@ -7,7 +7,7 @@ import pytest from tests.util import get_test_data -from vis4d.config.util.registry import get_config_by_name, register_config +from vis4d.config.registry import get_config_by_name, register_config class TestRegistry(unittest.TestCase): @@ -16,7 +16,7 @@ class TestRegistry(unittest.TestCase): def test_yaml(self) -> None: """Test reading a yaml config file.""" file = get_test_data( - "config_test/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.yaml" + "zoo_test/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.yaml" ) # Config can be resolved diff --git a/tests/config/show_connection_test.py b/tests/config/show_connection_test.py index 64ea7b670..1011020f7 100644 --- a/tests/config/show_connection_test.py +++ b/tests/config/show_connection_test.py @@ -2,7 +2,8 @@ import unittest -from tests.zoo.util import content_equal, get_config_for_name +from tests.util import content_equal, get_test_file +from tests.zoo.util import get_config_for_name from vis4d.config import instantiate_classes from vis4d.config.show_connection import prints_datagraph_for_config @@ -10,8 +11,6 @@ class TestShowConfig(unittest.TestCase): """Tests the content of the provided configs for Show.""" - gt_config_path = "tests/vis4d-test-data/config_test" - def test_show_frcnn(self) -> None: """Test the config for faster_rcnn_coco.py. @@ -36,11 +35,7 @@ def test_show_frcnn(self) -> None: model, train_data_connector, test_data_connector, loss, callbacks ) - with open( - f"{self.gt_config_path}/connection.txt", - "r", - encoding="UTF-8", - ) as f: + with open(get_test_file("connection.txt"), "r", encoding="UTF-8") as f: gt_dg = f.read() self.assertTrue(content_equal(dg, gt_dg)) diff --git a/tests/config/sweep/__init__.py b/tests/config/sweep/__init__.py deleted file mode 100644 index 6dfff8494..000000000 --- a/tests/config/sweep/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Vis4d Sweep Config Tests.""" diff --git a/tests/config/sweep/sweep_test.py b/tests/config/sweep_test.py similarity index 97% rename from tests/config/sweep/sweep_test.py rename to tests/config/sweep_test.py index 9b788a0db..29ed1847e 100644 --- a/tests/config/sweep/sweep_test.py +++ b/tests/config/sweep_test.py @@ -6,8 +6,8 @@ from ml_collections import ConfigDict from vis4d.config.replicator import replicate_config -from vis4d.config.typing import ExperimentConfig -from vis4d.config.util.sweep import grid_search +from vis4d.config.sweep import grid_search +from vis4d.zoo.typing import ExperimentConfig class TestSweep(unittest.TestCase): diff --git a/tests/config/testcases/connection.txt b/tests/config/testcases/connection.txt new file mode 100644 index 000000000..6df3a72da --- /dev/null +++ b/tests/config/testcases/connection.txt @@ -0,0 +1,62 @@ +=================================== += Training Loop = +=================================== + -------------- + -boxes2d | | *boxes2d  +-boxes2d_classes | | *boxes2d_classes + -images | Train Data | *images  + -input_hw | | *input_hw  + -------------- + -------------- + boxes2d* | | 

-proposals  + boxes2d_classes* | | 

-roi  + images* | | *

-rpn  + input_hw* | FasterRCNN | 

-sampled_proposals  + original_hw | | 

-sampled_target_indices +  | | 

-sampled_targets  + -------------- + ----------- +

-rpn.cls* | | cls_outs  + -input_hw | | images_hw  +

-rpn.box* | RPNLoss | reg_outs  + -boxes2d | | target_boxes + ----------- + ------------ +

-sampled_proposals.boxes | | boxes  +

-sampled_targets.labels | | boxes_mask  +

-roi.cls_score | | class_outs  +

-roi.bbox_pred | RCNNLoss | regression_outs +

-sampled_targets.boxes | | target_boxes  +

-sampled_targets.classes | | target_classes  + ------------ +=================================== += Testing Loop = +=================================== + ------------- + -images | | *images  + -input_hw | Test Data | *input_hw  +-original_hw | | *original_hw + ------------- + -------------- + boxes2d | | 

-boxes  +boxes2d_classes | | 

-class_ids + images* | FasterRCNN | 

-scores  + input_hw* | |   + original_hw* | |   + -------------- +=================================== += Callbacks = +=================================== + ------------------------- +

-boxes | | *boxes  +

-class_ids | | *class_ids  + -sample_names | BoundingBoxVisualizer | *image_names +-original_images | | *images  +

-scores | | *scores  + ------------------------- + ----------------------- +-sample_names | | coco_image_id +

-boxes | | pred_boxes  +

-class_ids | COCODetectEvaluator | pred_classes  +

-scores | | pred_scores  + ----------------------- diff --git a/tests/engine/callbacks/evaluator_test.py b/tests/engine/callbacks/evaluator_test.py index 20666adf6..40dfbf457 100644 --- a/tests/engine/callbacks/evaluator_test.py +++ b/tests/engine/callbacks/evaluator_test.py @@ -7,11 +7,11 @@ import torch from tests.util import MockModel, get_test_data -from vis4d.config.common.datasets.coco import CONN_COCO_MASK_EVAL from vis4d.data.const import CommonKeys as K from vis4d.engine.callbacks import EvaluatorCallback, TrainerState from vis4d.engine.connectors import CallbackConnector from vis4d.eval.coco import COCODetectEvaluator +from vis4d.zoo.base.datasets.coco import CONN_COCO_MASK_EVAL class TestEvaluatorCallback(unittest.TestCase): diff --git a/tests/engine/callbacks/visualizer_test.py b/tests/engine/callbacks/visualizer_test.py index 897dedf28..2a4f590e7 100644 --- a/tests/engine/callbacks/visualizer_test.py +++ b/tests/engine/callbacks/visualizer_test.py @@ -7,11 +7,11 @@ import torch from tests.util import MOCKLOSS, MockModel -from vis4d.config.default.data_connectors import CONN_BBOX_2D_VIS from vis4d.data.const import CommonKeys as K from vis4d.engine.callbacks import TrainerState, VisualizerCallback from vis4d.engine.connectors import CallbackConnector from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_VIS class TestVisualizerCallback(unittest.TestCase): diff --git a/tests/engine/optim/optimizer_test.py b/tests/engine/optim/optimizer_test.py index 020882462..256965c8f 100644 --- a/tests/engine/optim/optimizer_test.py +++ b/tests/engine/optim/optimizer_test.py @@ -12,9 +12,9 @@ from tests.util import MockModel from vis4d.config import class_config -from vis4d.config.typing import LrSchedulerConfig, ParamGroupCfg -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.engine.optim import LRSchedulerWrapper, PolyLR, set_up_optimizers +from vis4d.zoo.base import get_lr_scheduler_cfg, get_optimizer_cfg +from vis4d.zoo.typing import LrSchedulerConfig, ParamGroupCfg def get_optimizer( diff --git a/tests/model/detect/mask_rcnn_test.py b/tests/model/detect/mask_rcnn_test.py index b39c038f0..4b5ff954a 100644 --- a/tests/model/detect/mask_rcnn_test.py +++ b/tests/model/detect/mask_rcnn_test.py @@ -7,11 +7,6 @@ from tests.util import get_test_data, get_test_file from vis4d.common.ckpt import load_model_checkpoint -from vis4d.config.common.models.mask_rcnn import ( - CONN_MASK_HEAD_LOSS_2D, - CONN_ROI_LOSS_2D, - CONN_RPN_LOSS_2D, -) from vis4d.data.const import CommonKeys as K from vis4d.data.datasets import COCO from vis4d.engine.connectors import LossConnector @@ -29,6 +24,11 @@ ) from vis4d.op.detect.rcnn import RCNNLoss, get_default_rcnn_box_codec from vis4d.op.detect.rpn import RPNLoss, get_default_rpn_box_codec +from vis4d.zoo.base.models.mask_rcnn import ( + CONN_MASK_HEAD_LOSS_2D, + CONN_ROI_LOSS_2D, + CONN_RPN_LOSS_2D, +) from .faster_rcnn_test import get_test_dataloader, get_train_dataloader diff --git a/tests/pl/data_module_test.py b/tests/pl/data_module_test.py index a7c3d4bab..c4ce4a023 100644 --- a/tests/pl/data_module_test.py +++ b/tests/pl/data_module_test.py @@ -7,8 +7,8 @@ from torch.utils.data.dataloader import DataLoader from tests.util import get_test_data -from vis4d.config.common.datasets.coco import get_coco_detection_cfg from vis4d.pl.data_module import DataModule +from vis4d.zoo.base.datasets.coco import get_coco_detection_cfg class DataModuleTest(unittest.TestCase): diff --git a/tests/pl/trainer_test.py b/tests/pl/trainer_test.py index 6cca70a86..2608983ff 100644 --- a/tests/pl/trainer_test.py +++ b/tests/pl/trainer_test.py @@ -12,7 +12,6 @@ from tests.util import get_test_data from vis4d.config import class_config -from vis4d.config.util import get_optimizer_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.datasets import COCO from vis4d.data.loader import DataPipe, build_train_dataloader @@ -38,6 +37,7 @@ from vis4d.pl.callbacks import CallbackWrapper, LRSchedulerCallback from vis4d.pl.trainer import PLTrainer from vis4d.pl.training_module import TrainingModule +from vis4d.zoo.base import get_optimizer_cfg def seg_pipeline(data: list[DictData]) -> DictData: diff --git a/tests/util.py b/tests/util.py index 7b919fb54..33fe36dbb 100644 --- a/tests/util.py +++ b/tests/util.py @@ -366,3 +366,43 @@ def forward( # type: ignore ).sum() } return {} + + +def content_equal( + content1: str, content2: str, ignored_props: list[str] | None = None +) -> bool: + """Compare two strings line by line. + + Args: + content1 (str): First file content + content2 (str): Second file content + ignored_props (list[str], optional): List of properties to ignore. + All lines matching any of these properties (followed by a ':') + will be ignored. Defaults to []. + """ + if ignored_props is None: + ignored_props = [] + + lines1 = content1.splitlines() + lines2 = content2.splitlines() + if len(lines1) != len(lines2): + print("File length mismatch:", len(lines1), "!=", len(lines2)) + return False + + for line_id, line1 in enumerate(lines1): + skip = False + for prop in ignored_props: + # Append `:` to avoid matching a property that is a substring of + # another property + prop = prop + ":" + if prop in line1 and prop in lines2[line_id]: + skip = True + continue # ignore these lines + + if not skip and line1 != lines2[line_id]: + print( + "Line mismatch #", line_id, ":", line1, "!=", lines2[line_id] + ) + return False + + return True diff --git a/tests/zoo/base/__init__.py b/tests/zoo/base/__init__.py new file mode 100644 index 000000000..f2a2999ec --- /dev/null +++ b/tests/zoo/base/__init__.py @@ -0,0 +1 @@ +"""Model zoo base test.""" diff --git a/tests/config/common/datasets/__init__.py b/tests/zoo/base/datasets/__init__.py similarity index 100% rename from tests/config/common/datasets/__init__.py rename to tests/zoo/base/datasets/__init__.py diff --git a/tests/config/common/datasets/bdd100k/__init__.py b/tests/zoo/base/datasets/bdd100k/__init__.py similarity index 100% rename from tests/config/common/datasets/bdd100k/__init__.py rename to tests/zoo/base/datasets/bdd100k/__init__.py diff --git a/tests/config/common/datasets/bdd100k/detect_test.py b/tests/zoo/base/datasets/bdd100k/detect_test.py similarity index 97% rename from tests/config/common/datasets/bdd100k/detect_test.py rename to tests/zoo/base/datasets/bdd100k/detect_test.py index 8c10e98c5..e4df5b71a 100644 --- a/tests/config/common/datasets/bdd100k/detect_test.py +++ b/tests/zoo/base/datasets/bdd100k/detect_test.py @@ -4,11 +4,11 @@ from tests.util import get_test_data from vis4d.config import instantiate_classes -from vis4d.config.common.datasets.bdd100k.detect import ( +from vis4d.data.const import CommonKeys as K +from vis4d.zoo.base.datasets.bdd100k.detect import ( get_test_dataloader, get_train_dataloader, ) -from vis4d.data.const import CommonKeys as K class TestDetPreprocessing(unittest.TestCase): diff --git a/tests/config/common/datasets/bdd100k/sem_seg_test.py b/tests/zoo/base/datasets/bdd100k/sem_seg_test.py similarity index 97% rename from tests/config/common/datasets/bdd100k/sem_seg_test.py rename to tests/zoo/base/datasets/bdd100k/sem_seg_test.py index f66b4ca01..72c904187 100644 --- a/tests/config/common/datasets/bdd100k/sem_seg_test.py +++ b/tests/zoo/base/datasets/bdd100k/sem_seg_test.py @@ -4,11 +4,11 @@ from tests.util import get_test_data from vis4d.config import instantiate_classes -from vis4d.config.common.datasets.bdd100k.sem_seg import ( +from vis4d.data.const import CommonKeys as K +from vis4d.zoo.base.datasets.bdd100k.sem_seg import ( get_test_dataloader, get_train_dataloader, ) -from vis4d.data.const import CommonKeys as K class TestSegPreprocessing(unittest.TestCase): diff --git a/tests/config/common/datasets/coco/__init__.py b/tests/zoo/base/datasets/coco/__init__.py similarity index 100% rename from tests/config/common/datasets/coco/__init__.py rename to tests/zoo/base/datasets/coco/__init__.py diff --git a/tests/config/common/datasets/coco/detection_test.py b/tests/zoo/base/datasets/coco/detection_test.py similarity index 97% rename from tests/config/common/datasets/coco/detection_test.py rename to tests/zoo/base/datasets/coco/detection_test.py index 748494bd7..5a9b292a3 100644 --- a/tests/config/common/datasets/coco/detection_test.py +++ b/tests/zoo/base/datasets/coco/detection_test.py @@ -9,8 +9,8 @@ from tests.util import get_test_data from vis4d.config import class_config, instantiate_classes -from vis4d.config.common.datasets.coco import get_coco_detection_cfg from vis4d.data.datasets.coco import COCO +from vis4d.zoo.base.datasets.coco import get_coco_detection_cfg class TestDataloaderConfig(unittest.TestCase): diff --git a/tests/config/common/datasets/nuscenes/__init__.py b/tests/zoo/base/datasets/nuscenes/__init__.py similarity index 100% rename from tests/config/common/datasets/nuscenes/__init__.py rename to tests/zoo/base/datasets/nuscenes/__init__.py diff --git a/tests/config/common/datasets/nuscenes/nuscenes_mono_test.py b/tests/zoo/base/datasets/nuscenes/nuscenes_mono_test.py similarity index 92% rename from tests/config/common/datasets/nuscenes/nuscenes_mono_test.py rename to tests/zoo/base/datasets/nuscenes/nuscenes_mono_test.py index 3215f2de3..15bd5521c 100644 --- a/tests/config/common/datasets/nuscenes/nuscenes_mono_test.py +++ b/tests/zoo/base/datasets/nuscenes/nuscenes_mono_test.py @@ -8,8 +8,8 @@ from tests.util import get_test_data from vis4d.config import instantiate_classes -from vis4d.config.common.datasets.nuscenes import get_nusc_mono_mini_train_cfg from vis4d.data.datasets.nuscenes_mono import NuScenesMono +from vis4d.zoo.base.datasets.nuscenes import get_nusc_mono_mini_train_cfg class TestNuscenesConfig(unittest.TestCase): diff --git a/tests/config/common/datasets/nuscenes/nuscenes_test.py b/tests/zoo/base/datasets/nuscenes/nuscenes_test.py similarity index 96% rename from tests/config/common/datasets/nuscenes/nuscenes_test.py rename to tests/zoo/base/datasets/nuscenes/nuscenes_test.py index 631713c97..78743fb9f 100644 --- a/tests/config/common/datasets/nuscenes/nuscenes_test.py +++ b/tests/zoo/base/datasets/nuscenes/nuscenes_test.py @@ -8,11 +8,11 @@ from tests.util import get_test_data from vis4d.config import instantiate_classes -from vis4d.config.common.datasets.nuscenes import ( +from vis4d.data.datasets.nuscenes import NuScenes +from vis4d.zoo.base.datasets.nuscenes import ( get_nusc_mini_train_cfg, get_nusc_mini_val_cfg, ) -from vis4d.data.datasets.nuscenes import NuScenes class TestNuscenesConfig(unittest.TestCase): diff --git a/tests/config/common/datasets/shift/__init__.py b/tests/zoo/base/datasets/shift/__init__.py similarity index 100% rename from tests/config/common/datasets/shift/__init__.py rename to tests/zoo/base/datasets/shift/__init__.py diff --git a/tests/config/common/datasets/shift/test_shift_tasks.py b/tests/zoo/base/datasets/shift/test_shift_tasks.py similarity index 99% rename from tests/config/common/datasets/shift/test_shift_tasks.py rename to tests/zoo/base/datasets/shift/test_shift_tasks.py index 5d45e0a06..536d44956 100644 --- a/tests/config/common/datasets/shift/test_shift_tasks.py +++ b/tests/zoo/base/datasets/shift/test_shift_tasks.py @@ -9,15 +9,15 @@ from tests.util import get_test_data from vis4d.config import class_config, instantiate_classes -from vis4d.config.common.datasets.shift import ( +from vis4d.data.const import CommonKeys as K +from vis4d.data.io import HDF5Backend +from vis4d.zoo.base.datasets.shift import ( get_shift_depth_est_config, get_shift_det_config, get_shift_instance_seg_config, get_shift_multitask_2d_config, get_shift_sem_seg_config, ) -from vis4d.data.const import CommonKeys as K -from vis4d.data.io import HDF5Backend class TestMultiViewDataloaderConfig(unittest.TestCase): diff --git a/tests/zoo/bdd100k/faster_rcnn_test.py b/tests/zoo/bdd100k/faster_rcnn_test.py index 9b8410b80..a9b7bfa67 100644 --- a/tests/zoo/bdd100k/faster_rcnn_test.py +++ b/tests/zoo/bdd100k/faster_rcnn_test.py @@ -9,7 +9,7 @@ class TestFasterRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Faster R-CNN.""" config_prefix = "bdd100k.faster_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/bdd100k/faster_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/bdd100k/faster_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_1x(self) -> None: diff --git a/tests/zoo/bdd100k/mask_rcnn_test.py b/tests/zoo/bdd100k/mask_rcnn_test.py index deda27238..e5484ca54 100644 --- a/tests/zoo/bdd100k/mask_rcnn_test.py +++ b/tests/zoo/bdd100k/mask_rcnn_test.py @@ -9,7 +9,7 @@ class TestMaskRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Mask R-CNN.""" config_prefix = "bdd100k.mask_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/bdd100k/mask_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/bdd100k/mask_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_1x(self) -> None: diff --git a/tests/zoo/bdd100k/qdtrack_test.py b/tests/zoo/bdd100k/qdtrack_test.py index 612e947dd..71f665004 100644 --- a/tests/zoo/bdd100k/qdtrack_test.py +++ b/tests/zoo/bdd100k/qdtrack_test.py @@ -9,7 +9,7 @@ class TestQDTrackConfig(unittest.TestCase): """Tests the content of the provided configs for QDTrack.""" config_prefix = "bdd100k.qdtrack" - gt_config_path = "tests/vis4d-test-data/config_test/bdd100k/qdtrack" + gt_config_path = "tests/vis4d-test-data/zoo_test/bdd100k/qdtrack" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_frcnn_r50_fpn_1x_bdd100k(self) -> None: diff --git a/tests/zoo/bdd100k/semantic_fpn_test.py b/tests/zoo/bdd100k/semantic_fpn_test.py index 959607da9..3b9e90dae 100644 --- a/tests/zoo/bdd100k/semantic_fpn_test.py +++ b/tests/zoo/bdd100k/semantic_fpn_test.py @@ -9,7 +9,7 @@ class TestSemanticFPNConfig(unittest.TestCase): """Tests the content of the provided configs for Semantic-FPN.""" config_prefix = "bdd100k.semantic_fpn" - gt_config_path = "tests/vis4d-test-data/config_test/bdd100k/semantic_fpn" + gt_config_path = "tests/vis4d-test-data/zoo_test/bdd100k/semantic_fpn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_40k(self) -> None: diff --git a/tests/zoo/bevformer_test.py b/tests/zoo/bevformer_test.py index 70f921746..c6ec93195 100644 --- a/tests/zoo/bevformer_test.py +++ b/tests/zoo/bevformer_test.py @@ -9,7 +9,7 @@ class TestBEVFormerConfig(unittest.TestCase): """Tests the content of the provided configs.""" config_prefix = "bevformer" - gt_config_path = "tests/vis4d-test-data/config_test/bevformer" + gt_config_path = "tests/vis4d-test-data/zoo_test/bevformer" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_bevformer_base(self) -> None: diff --git a/tests/zoo/cc_3dt_test.py b/tests/zoo/cc_3dt_test.py index abd4cf43c..93f04e18a 100644 --- a/tests/zoo/cc_3dt_test.py +++ b/tests/zoo/cc_3dt_test.py @@ -9,7 +9,7 @@ class TestCC3DTConfig(unittest.TestCase): """Tests the content of the provided configs.""" config_prefix = "cc_3dt" - gt_config_path = "tests/vis4d-test-data/config_test/cc_3dt" + gt_config_path = "tests/vis4d-test-data/zoo_test/cc_3dt" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_frcnn_r50_fpn_kf3d_12e_nusc(self) -> None: diff --git a/tests/zoo/faster_rcnn_test.py b/tests/zoo/faster_rcnn_test.py index 54d7a66b6..f5404d1a6 100644 --- a/tests/zoo/faster_rcnn_test.py +++ b/tests/zoo/faster_rcnn_test.py @@ -9,7 +9,7 @@ class TestFasterRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Faster-RCNN.""" config_prefix = "faster_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/faster_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/faster_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_coco(self) -> None: diff --git a/tests/zoo/fcn_resnet_test.py b/tests/zoo/fcn_resnet_test.py index edb666c3c..f7acd4384 100644 --- a/tests/zoo/fcn_resnet_test.py +++ b/tests/zoo/fcn_resnet_test.py @@ -9,7 +9,7 @@ class TestFCNResNetConfig(unittest.TestCase): """Tests the content of the provided configs for FCN ResNet.""" config_prefix = "fcn_resnet" - gt_config_path = "tests/vis4d-test-data/config_test/fcn_resnet" + gt_config_path = "tests/vis4d-test-data/zoo_test/fcn_resnet" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_coco(self) -> None: diff --git a/tests/zoo/mask_rcnn_test.py b/tests/zoo/mask_rcnn_test.py index da2782814..c2ce2cf10 100644 --- a/tests/zoo/mask_rcnn_test.py +++ b/tests/zoo/mask_rcnn_test.py @@ -9,7 +9,7 @@ class TestMaskRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Mask-RCNN.""" config_prefix = "mask_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/mask_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/mask_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_coco(self) -> None: diff --git a/tests/zoo/qdtrack_test.py b/tests/zoo/qdtrack_test.py index 2d391e03f..6861d0a9a 100644 --- a/tests/zoo/qdtrack_test.py +++ b/tests/zoo/qdtrack_test.py @@ -9,7 +9,7 @@ class TestQDTrackConfig(unittest.TestCase): """Tests the content of the provided configs for QDTrack.""" config_prefix = "qdtrack" - gt_config_path = "tests/vis4d-test-data/config_test/qdtrack" + gt_config_path = "tests/vis4d-test-data/zoo_test/qdtrack" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_frcnn_r50_fpn_augs_1x_bdd100k(self) -> None: diff --git a/tests/zoo/retinanet_test.py b/tests/zoo/retinanet_test.py index 5a929fee7..01d448dc9 100644 --- a/tests/zoo/retinanet_test.py +++ b/tests/zoo/retinanet_test.py @@ -9,7 +9,7 @@ class TestRetinaNetConfig(unittest.TestCase): """Tests the content of the provided configs for RetinaNet.""" config_prefix = "retinanet" - gt_config_path = "tests/vis4d-test-data/config_test/retinanet" + gt_config_path = "tests/vis4d-test-data/zoo_test/retinanet" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_coco(self) -> None: diff --git a/tests/zoo/shift/faster_rcnn_test.py b/tests/zoo/shift/faster_rcnn_test.py index 3641becd8..5d5ec1645 100644 --- a/tests/zoo/shift/faster_rcnn_test.py +++ b/tests/zoo/shift/faster_rcnn_test.py @@ -9,7 +9,7 @@ class TestFasterRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Faster R-CNN.""" config_prefix = "shift.faster_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/shift/faster_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/shift/faster_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_12e(self) -> None: diff --git a/tests/zoo/shift/mask_rcnn_test.py b/tests/zoo/shift/mask_rcnn_test.py index b52fe4a86..93308e43a 100644 --- a/tests/zoo/shift/mask_rcnn_test.py +++ b/tests/zoo/shift/mask_rcnn_test.py @@ -9,7 +9,7 @@ class TestMaskRCNNConfig(unittest.TestCase): """Tests the content of the provided configs for Mask R-CNN.""" config_prefix = "shift.mask_rcnn" - gt_config_path = "tests/vis4d-test-data/config_test/shift/mask_rcnn" + gt_config_path = "tests/vis4d-test-data/zoo_test/shift/mask_rcnn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_12e(self) -> None: diff --git a/tests/zoo/shift/semantic_fpn_test.py b/tests/zoo/shift/semantic_fpn_test.py index 2445aac9f..13ede1bf8 100644 --- a/tests/zoo/shift/semantic_fpn_test.py +++ b/tests/zoo/shift/semantic_fpn_test.py @@ -9,7 +9,7 @@ class TestSemanticFPNConfig(unittest.TestCase): """Tests the content of the provided configs for Semantic-FPN.""" config_prefix = "shift.semantic_fpn" - gt_config_path = "tests/vis4d-test-data/config_test/shift/semantic_fpn" + gt_config_path = "tests/vis4d-test-data/zoo_test/shift/semantic_fpn" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_r50_fpn_160k(self) -> None: diff --git a/tests/zoo/util.py b/tests/zoo/util.py index 4e2524ecd..5aff0e514 100644 --- a/tests/zoo/util.py +++ b/tests/zoo/util.py @@ -4,7 +4,8 @@ import importlib -from vis4d.config.typing import ExperimentConfig +from tests.util import content_equal +from vis4d.zoo.typing import ExperimentConfig def get_config_for_name(config_name: str) -> ExperimentConfig: @@ -14,46 +15,6 @@ def get_config_for_name(config_name: str) -> ExperimentConfig: return module.get_config() -def content_equal( - content1: str, content2: str, ignored_props: list[str] | None = None -) -> bool: - """Compare two strings line by line. - - Args: - content1 (str): First file content - content2 (str): Second file content - ignored_props (list[str], optional): List of properties to ignore. - All lines matching any of these properties (followed by a ':') - will be ignored. Defaults to []. - """ - if ignored_props is None: - ignored_props = [] - - lines1 = content1.splitlines() - lines2 = content2.splitlines() - if len(lines1) != len(lines2): - print("File length mismatch:", len(lines1), "!=", len(lines2)) - return False - - for line_id, line1 in enumerate(lines1): - skip = False - for prop in ignored_props: - # Append `:` to avoid matching a property that is a substring of - # another property - prop = prop + ":" - if prop in line1 and prop in lines2[line_id]: - skip = True - continue # ignore these lines - - if not skip and line1 != lines2[line_id]: - print( - "Line mismatch #", line_id, ":", line1, "!=", lines2[line_id] - ) - return False - - return True - - def compare_configs( cfg_cur: str, cfg_gt: str, varying_keys: list[str] | None = None ) -> bool: diff --git a/tests/zoo/vit_imagenet_test.py b/tests/zoo/vit_imagenet_test.py index 52bcb9315..6e0c0ac6e 100644 --- a/tests/zoo/vit_imagenet_test.py +++ b/tests/zoo/vit_imagenet_test.py @@ -9,7 +9,7 @@ class TestViTConfig(unittest.TestCase): """Tests the content of the provided configs for ViT.""" config_prefix = "vit" - gt_config_path = "tests/vis4d-test-data/config_test/vit" + gt_config_path = "tests/vis4d-test-data/zoo_test/vit" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_small_imagenet(self) -> None: diff --git a/tests/zoo/yolox_test.py b/tests/zoo/yolox_test.py index 029a3bcde..f3d44a015 100644 --- a/tests/zoo/yolox_test.py +++ b/tests/zoo/yolox_test.py @@ -9,7 +9,7 @@ class TestYOLOXConfig(unittest.TestCase): """Tests the content of the provided configs for YOLOX.""" config_prefix = "yolox" - gt_config_path = "tests/vis4d-test-data/config_test/yolox" + gt_config_path = "tests/vis4d-test-data/zoo_test/yolox" varying_keys = ["save_prefix", "output_dir", "version", "timestamp"] def test_yolox_tiny_300e_coco(self) -> None: diff --git a/vis4d/common/typing.py b/vis4d/common/typing.py index 5ebc0bfa4..718581705 100644 --- a/vis4d/common/typing.py +++ b/vis4d/common/typing.py @@ -26,6 +26,8 @@ NDArrayI64 = npt.NDArray[np.int64] NDArrayI32 = npt.NDArray[np.int32] NDArrayUI8 = npt.NDArray[np.uint8] +NDArrayUI16 = npt.NDArray[np.uint16] +NDArrayUI32 = npt.NDArray[np.uint32] NDArrayBool = npt.NDArray[np.bool_] NDArrayFloat = Union[NDArrayF32, NDArrayF64] NDArrayInt = Union[NDArrayI32, NDArrayI64] diff --git a/vis4d/common/util.py b/vis4d/common/util.py index 49845040d..75d51ce4c 100644 --- a/vis4d/common/util.py +++ b/vis4d/common/util.py @@ -5,9 +5,10 @@ import numpy as np import torch +from packaging import version from .imports import is_torch_tf32_available -from .logging import rank_zero_warn +from .logging import rank_zero_info, rank_zero_warn def create_did_you_mean_msg(keys: list[str], query: str) -> str: @@ -34,22 +35,40 @@ def create_did_you_mean_msg(keys: list[str], query: str) -> str: return msg -def set_tf32(use_tf32: bool = False) -> None: # pragma: no cover +def set_tf32(use_tf32: bool, precision: str) -> None: # pragma: no cover """Set torch TF32. Args: - use_tf32: Whether to use torch TF32. + use_tf32: Whether to use torch TF32. Details: + https://pytorch.org/docs/stable/notes/cuda.html#tf32-on-ampere + precision: Internal precision of float32 matrix multiplications. + Details: https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision # pylint: disable=line-too-long """ - if is_torch_tf32_available(): # pragma: no cover - if use_tf32: - rank_zero_warn( - "Torch TF32 is available and turned on by default! " - + "It might harm the performance due to the precision. " - + "You can turn it off by setting trainer.use_tf32=False." - ) + if use_tf32: # pragma: no cover + rank_zero_info( + "Using Torch TF32. " + + "It might harm the performance due to the precision. " + + "You can turn it off by setting config.use_tf32=False." + ) + if not is_torch_tf32_available(): + rank_zero_warn("Torch TF32 is not available.") + elif ( + version.parse("1.11") + >= version.parse(torch.__version__) + >= version.parse("1.7") + ): + rank_zero_info("Torch TF32 is turned on by default!") else: - torch.backends.cuda.matmul.allow_tf32 = False - torch.backends.cudnn.allow_tf32 = False + rank_zero_info("Turn on Torch TF32 on matmul.") + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + else: + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + # Control the precision of matmul operations. + # Equivalent to setting torch.backends.cuda.matmul.allow_tf32. + torch.set_float32_matmul_precision(precision) def init_random_seed() -> int: diff --git a/vis4d/config/common/__init__.py b/vis4d/config/common/__init__.py deleted file mode 100644 index 6ee9e3952..000000000 --- a/vis4d/config/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Common config.""" diff --git a/vis4d/config/common/datasets/__init__.py b/vis4d/config/common/datasets/__init__.py deleted file mode 100644 index eda9d4bec..000000000 --- a/vis4d/config/common/datasets/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Common dataset config.""" diff --git a/vis4d/config/common/datasets/imagenet/__init__.py b/vis4d/config/common/datasets/imagenet/__init__.py deleted file mode 100644 index 0149b0d4a..000000000 --- a/vis4d/config/common/datasets/imagenet/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""ImageNet dataset config.""" - -from .cls import CONN_IMAGENET_CLS_EVAL, get_imagenet_cls_cfg - -__all__ = ["get_imagenet_cls_cfg", "CONN_IMAGENET_CLS_EVAL"] diff --git a/vis4d/config/common/models/__init__.py b/vis4d/config/common/models/__init__.py deleted file mode 100644 index c3cd382f5..000000000 --- a/vis4d/config/common/models/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Common models config.""" - -from .faster_rcnn import get_faster_rcnn_cfg -from .mask_rcnn import get_mask_rcnn_cfg - -__all__ = ["get_faster_rcnn_cfg", "get_mask_rcnn_cfg"] diff --git a/vis4d/config/default/__init__.py b/vis4d/config/default/__init__.py deleted file mode 100644 index 298bcddc5..000000000 --- a/vis4d/config/default/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Config default.""" - -from .pl_trainer import get_default_pl_trainer_cfg -from .runtime import get_default_callbacks_cfg, get_default_cfg - -__all__ = [ - "get_default_cfg", - "get_default_callbacks_cfg", - "get_default_pl_trainer_cfg", -] diff --git a/vis4d/config/util/registry.py b/vis4d/config/registry.py similarity index 99% rename from vis4d/config/util/registry.py rename to vis4d/config/registry.py index 628c1b958..2d39870a9 100644 --- a/vis4d/config/util/registry.py +++ b/vis4d/config/registry.py @@ -19,7 +19,7 @@ from vis4d.zoo import AVAILABLE_MODELS MODEL_ZOO_FOLDER = str( - (pathlib.Path(os.path.dirname(__file__)) / ".." / ".." / "zoo").resolve() + (pathlib.Path(os.path.dirname(__file__)) / ".." / "zoo").resolve() ) # Paths that are used to search for config files. diff --git a/vis4d/config/util/sweep.py b/vis4d/config/sweep.py similarity index 100% rename from vis4d/config/util/sweep.py rename to vis4d/config/sweep.py diff --git a/vis4d/engine/experiment.py b/vis4d/engine/experiment.py index c7543b12a..eae6dadbc 100644 --- a/vis4d/engine/experiment.py +++ b/vis4d/engine/experiment.py @@ -31,7 +31,7 @@ from vis4d.common.slurm import init_dist_slurm from vis4d.common.util import init_random_seed, set_random_seed, set_tf32 from vis4d.config import instantiate_classes -from vis4d.config.typing import ExperimentConfig +from vis4d.zoo.typing import ExperimentConfig from .optim import set_up_optimizers from .parser import pprints_config @@ -110,7 +110,8 @@ def run_experiment( rank_zero_info("Environment info: %s", get_pretty_env_info()) # PyTorch Setting - set_tf32(config.use_tf32) + set_tf32(config.use_tf32, config.tf32_matmul_precision) + torch.hub.set_dir(f"{config.work_dir}/.cache/torch/hub") torch.backends.cudnn.benchmark = config.benchmark if show_config: diff --git a/vis4d/engine/optim/optimizer.py b/vis4d/engine/optim/optimizer.py index 33629ee5b..23adf9832 100644 --- a/vis4d/engine/optim/optimizer.py +++ b/vis4d/engine/optim/optimizer.py @@ -13,7 +13,7 @@ from vis4d.common.logging import rank_zero_info from vis4d.config import instantiate_classes -from vis4d.config.typing import OptimizerConfig, ParamGroupCfg +from vis4d.zoo.typing import OptimizerConfig, ParamGroupCfg from .scheduler import LRSchedulerWrapper diff --git a/vis4d/engine/optim/scheduler.py b/vis4d/engine/optim/scheduler.py index 8d2c3a247..79effc206 100644 --- a/vis4d/engine/optim/scheduler.py +++ b/vis4d/engine/optim/scheduler.py @@ -9,7 +9,7 @@ from vis4d.common.typing import DictStrAny from vis4d.config import copy_and_resolve_references, instantiate_classes -from vis4d.config.typing import LrSchedulerConfig +from vis4d.zoo.typing import LrSchedulerConfig class LRSchedulerDict(TypedDict): diff --git a/vis4d/engine/parser.py b/vis4d/engine/parser.py index d77ae1099..03e2189db 100644 --- a/vis4d/engine/parser.py +++ b/vis4d/engine/parser.py @@ -17,7 +17,7 @@ ) from vis4d.config import copy_and_resolve_references -from vis4d.config.util.registry import get_config_by_name +from vis4d.config.registry import get_config_by_name class ConfigFileParser(flags.ArgumentParser): # type: ignore diff --git a/vis4d/engine/run.py b/vis4d/engine/run.py index 406ab3cb0..47512a3f6 100644 --- a/vis4d/engine/run.py +++ b/vis4d/engine/run.py @@ -8,7 +8,7 @@ from vis4d.common.logging import rank_zero_info from vis4d.config import instantiate_classes from vis4d.config.replicator import replicate_config -from vis4d.config.typing import ExperimentConfig +from vis4d.zoo.typing import ExperimentConfig from .experiment import run_experiment from .flag import _CKPT, _CONFIG, _GPUS, _RESUME, _SHOW_CONFIG, _SLURM, _SWEEP diff --git a/vis4d/pl/data_module.py b/vis4d/pl/data_module.py index de307a6e1..58dd8504d 100644 --- a/vis4d/pl/data_module.py +++ b/vis4d/pl/data_module.py @@ -6,8 +6,8 @@ from torch.utils.data import DataLoader from vis4d.config import instantiate_classes -from vis4d.config.typing import DataConfig from vis4d.data.typing import DictData +from vis4d.zoo.typing import DataConfig class DataModule(pl.LightningDataModule): diff --git a/vis4d/pl/run.py b/vis4d/pl/run.py index 60654ed14..e4c76bef2 100644 --- a/vis4d/pl/run.py +++ b/vis4d/pl/run.py @@ -5,6 +5,7 @@ import logging import os.path as osp +import torch from absl import app from lightning.fabric.utilities.exceptions import MisconfigurationException from lightning.pytorch import Callback @@ -14,7 +15,6 @@ from vis4d.common.logging import rank_zero_info, setup_logger from vis4d.common.util import set_tf32 from vis4d.config import instantiate_classes -from vis4d.config.typing import ExperimentConfig from vis4d.engine.callbacks.checkpoint import CheckpointCallback from vis4d.engine.flag import _CKPT, _CONFIG, _GPUS, _RESUME, _SHOW_CONFIG from vis4d.engine.parser import pprints_config @@ -22,6 +22,7 @@ from vis4d.pl.data_module import DataModule from vis4d.pl.trainer import PLTrainer from vis4d.pl.training_module import TrainingModule +from vis4d.zoo.typing import ExperimentConfig def main(argv: ArgsType) -> None: @@ -46,7 +47,8 @@ def main(argv: ArgsType) -> None: rank_zero_info("Environment info: %s", get_pretty_env_info()) # PyTorch Setting - set_tf32(config.use_tf32) + set_tf32(config.use_tf32, config.tf32_matmul_precision) + torch.hub.set_dir(f"{config.work_dir}/.cache/torch/hub") # Setup device if num_gpus > 0: diff --git a/vis4d/pl/training_module.py b/vis4d/pl/training_module.py index 8692ebc99..79ba8a2c1 100644 --- a/vis4d/pl/training_module.py +++ b/vis4d/pl/training_module.py @@ -15,11 +15,11 @@ from vis4d.common.typing import DictStrAny from vis4d.common.util import init_random_seed from vis4d.config import instantiate_classes -from vis4d.config.typing import OptimizerConfig from vis4d.data.typing import DictData from vis4d.engine.connectors import DataConnector from vis4d.engine.loss_module import LossModule from vis4d.engine.optim import LRSchedulerWrapper, set_up_optimizers +from vis4d.zoo.typing import OptimizerConfig class TrainingModule(pl.LightningModule): diff --git a/vis4d/state/track/qdtrack.py b/vis4d/state/track/qdtrack.py index fd27ce3a2..316608954 100644 --- a/vis4d/state/track/qdtrack.py +++ b/vis4d/state/track/qdtrack.py @@ -14,7 +14,7 @@ class Track(TypedDict): - """CC-3DT Track state. + """QDTrack Track state. Attributes: box (Tensor): In shape (4,) and contains x1, y1, x2, y2. diff --git a/vis4d/config/util/__init__.py b/vis4d/zoo/base/__init__.py similarity index 60% rename from vis4d/config/util/__init__.py rename to vis4d/zoo/base/__init__.py index 8fd238d50..b920b1c67 100644 --- a/vis4d/config/util/__init__.py +++ b/vis4d/zoo/base/__init__.py @@ -1,8 +1,10 @@ -"""Config utilities.""" +"""Model Zoo base.""" from .callable import get_callable_cfg from .dataloader import get_inference_dataloaders_cfg, get_train_dataloader_cfg from .optimizer import get_lr_scheduler_cfg, get_optimizer_cfg +from .pl_trainer import get_default_pl_trainer_cfg +from .runtime import get_default_callbacks_cfg, get_default_cfg __all__ = [ "get_callable_cfg", @@ -10,4 +12,7 @@ "get_inference_dataloaders_cfg", "get_optimizer_cfg", "get_lr_scheduler_cfg", + "get_default_cfg", + "get_default_callbacks_cfg", + "get_default_pl_trainer_cfg", ] diff --git a/vis4d/config/util/callable.py b/vis4d/zoo/base/callable.py similarity index 100% rename from vis4d/config/util/callable.py rename to vis4d/zoo/base/callable.py diff --git a/vis4d/config/default/data_connectors/__init__.py b/vis4d/zoo/base/data_connectors/__init__.py similarity index 92% rename from vis4d/config/default/data_connectors/__init__.py rename to vis4d/zoo/base/data_connectors/__init__.py index 952ea18ef..bb607050d 100644 --- a/vis4d/config/default/data_connectors/__init__.py +++ b/vis4d/zoo/base/data_connectors/__init__.py @@ -1,4 +1,4 @@ -"""Default data connection setups.""" +"""Base data connectors.""" from .common import CONN_IMAGES_TEST, CONN_IMAGES_TRAIN from .detection import CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BOX_LOSS_2D diff --git a/vis4d/config/default/data_connectors/cls.py b/vis4d/zoo/base/data_connectors/cls.py similarity index 88% rename from vis4d/config/default/data_connectors/cls.py rename to vis4d/zoo/base/data_connectors/cls.py index eca64ccd4..d76edfb5d 100644 --- a/vis4d/config/default/data_connectors/cls.py +++ b/vis4d/zoo/base/data_connectors/cls.py @@ -1,4 +1,4 @@ -"""Default data connectors for evaluators.""" +"""Data connectors for classification.""" from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import data_key, pred_key diff --git a/vis4d/config/default/data_connectors/common.py b/vis4d/zoo/base/data_connectors/common.py similarity index 82% rename from vis4d/config/default/data_connectors/common.py rename to vis4d/zoo/base/data_connectors/common.py index 58d1fd0d8..5738c731a 100644 --- a/vis4d/config/default/data_connectors/common.py +++ b/vis4d/zoo/base/data_connectors/common.py @@ -1,4 +1,4 @@ -"""Default data connectors for common tasks.""" +"""Data connectors for common tasks.""" from vis4d.data.const import CommonKeys as K diff --git a/vis4d/config/default/data_connectors/detection.py b/vis4d/zoo/base/data_connectors/detection.py similarity index 92% rename from vis4d/config/default/data_connectors/detection.py rename to vis4d/zoo/base/data_connectors/detection.py index 7bafec267..9ec2921da 100644 --- a/vis4d/config/default/data_connectors/detection.py +++ b/vis4d/zoo/base/data_connectors/detection.py @@ -1,4 +1,4 @@ -"""Default data connectors for detection.""" +"""Data connectors for detection.""" from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import data_key, pred_key diff --git a/vis4d/config/default/data_connectors/seg.py b/vis4d/zoo/base/data_connectors/seg.py similarity index 93% rename from vis4d/config/default/data_connectors/seg.py rename to vis4d/zoo/base/data_connectors/seg.py index cbc596650..b6899f725 100644 --- a/vis4d/config/default/data_connectors/seg.py +++ b/vis4d/zoo/base/data_connectors/seg.py @@ -1,4 +1,4 @@ -"""Default data connectors for evaluators.""" +"""Data connectors for segmentation.""" from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import data_key, pred_key diff --git a/vis4d/config/default/data_connectors/visualizers.py b/vis4d/zoo/base/data_connectors/visualizers.py similarity index 100% rename from vis4d/config/default/data_connectors/visualizers.py rename to vis4d/zoo/base/data_connectors/visualizers.py diff --git a/vis4d/config/util/dataloader.py b/vis4d/zoo/base/dataloader.py similarity index 100% rename from vis4d/config/util/dataloader.py rename to vis4d/zoo/base/dataloader.py diff --git a/vis4d/zoo/base/datasets/__init__.py b/vis4d/zoo/base/datasets/__init__.py new file mode 100644 index 000000000..085c5df6c --- /dev/null +++ b/vis4d/zoo/base/datasets/__init__.py @@ -0,0 +1 @@ +"""Model Zoo base datasets.""" diff --git a/vis4d/config/common/datasets/bdd100k/__init__.py b/vis4d/zoo/base/datasets/bdd100k/__init__.py similarity index 100% rename from vis4d/config/common/datasets/bdd100k/__init__.py rename to vis4d/zoo/base/datasets/bdd100k/__init__.py diff --git a/vis4d/config/common/datasets/bdd100k/detect.py b/vis4d/zoo/base/datasets/bdd100k/detect.py similarity index 98% rename from vis4d/config/common/datasets/bdd100k/detect.py rename to vis4d/zoo/base/datasets/bdd100k/detect.py index 7b5a1b230..78eff06a1 100644 --- a/vis4d/config/common/datasets/bdd100k/detect.py +++ b/vis4d/zoo/base/datasets/bdd100k/detect.py @@ -7,11 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets import BDD100K @@ -32,6 +27,11 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig CONN_BDD100K_DET_EVAL = { "frame_ids": data_key("frame_ids"), diff --git a/vis4d/config/common/datasets/bdd100k/sem_seg.py b/vis4d/zoo/base/datasets/bdd100k/sem_seg.py similarity index 98% rename from vis4d/config/common/datasets/bdd100k/sem_seg.py rename to vis4d/zoo/base/datasets/bdd100k/sem_seg.py index 7c8e71fc2..eb7e0616d 100644 --- a/vis4d/config/common/datasets/bdd100k/sem_seg.py +++ b/vis4d/zoo/base/datasets/bdd100k/sem_seg.py @@ -7,11 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.bdd100k import BDD100K @@ -33,6 +28,11 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig CONN_BDD100K_SEG_EVAL = { "data_names": data_key("sample_names"), diff --git a/vis4d/config/common/datasets/bdd100k/track.py b/vis4d/zoo/base/datasets/bdd100k/track.py similarity index 98% rename from vis4d/config/common/datasets/bdd100k/track.py rename to vis4d/zoo/base/datasets/bdd100k/track.py index 4aa6dc9c4..8a4190c97 100644 --- a/vis4d/config/common/datasets/bdd100k/track.py +++ b/vis4d/zoo/base/datasets/bdd100k/track.py @@ -5,11 +5,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets import BDD100K, bdd100k_track_map @@ -26,6 +21,11 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig CONN_BDD100K_TRACK_EVAL = { "frame_ids": data_key("frame_ids"), diff --git a/vis4d/config/common/datasets/coco/__init__.py b/vis4d/zoo/base/datasets/coco/__init__.py similarity index 100% rename from vis4d/config/common/datasets/coco/__init__.py rename to vis4d/zoo/base/datasets/coco/__init__.py diff --git a/vis4d/config/common/datasets/coco/detection.py b/vis4d/zoo/base/datasets/coco/detection.py similarity index 98% rename from vis4d/config/common/datasets/coco/detection.py rename to vis4d/zoo/base/datasets/coco/detection.py index c284f8c5e..fe99eaf74 100644 --- a/vis4d/config/common/datasets/coco/detection.py +++ b/vis4d/zoo/base/datasets/coco/detection.py @@ -7,11 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.coco import COCO @@ -32,6 +27,11 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig CONN_COCO_BBOX_EVAL = { "coco_image_id": data_key(K.sample_names), diff --git a/vis4d/config/common/datasets/coco/sem_seg.py b/vis4d/zoo/base/datasets/coco/sem_seg.py similarity index 98% rename from vis4d/config/common/datasets/coco/sem_seg.py rename to vis4d/zoo/base/datasets/coco/sem_seg.py index 52db7ded7..8bc2c9f58 100644 --- a/vis4d/config/common/datasets/coco/sem_seg.py +++ b/vis4d/zoo/base/datasets/coco/sem_seg.py @@ -7,11 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.coco import COCO @@ -27,6 +22,11 @@ ResizeSegMasks, ) from vis4d.data.transforms.to_tensor import ToTensor +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig def get_train_dataloader( diff --git a/vis4d/config/common/datasets/imagenet/cls.py b/vis4d/zoo/base/datasets/imagenet.py similarity index 98% rename from vis4d/config/common/datasets/imagenet/cls.py rename to vis4d/zoo/base/datasets/imagenet.py index 630b3547c..a1b4a7e5e 100644 --- a/vis4d/config/common/datasets/imagenet/cls.py +++ b/vis4d/zoo/base/datasets/imagenet.py @@ -7,8 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import get_train_dataloader_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.datasets.imagenet import ImageNet from vis4d.data.transforms.autoaugment import RandAug @@ -29,6 +27,8 @@ from vis4d.data.transforms.resize import GenResizeParameters, ResizeImages from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import get_train_dataloader_cfg +from vis4d.zoo.typing import DataConfig CONN_IMAGENET_CLS_EVAL = { "prediction": pred_key("probs"), diff --git a/vis4d/config/common/datasets/nuscenes/__init__.py b/vis4d/zoo/base/datasets/nuscenes/__init__.py similarity index 100% rename from vis4d/config/common/datasets/nuscenes/__init__.py rename to vis4d/zoo/base/datasets/nuscenes/__init__.py diff --git a/vis4d/config/common/datasets/nuscenes/nuscenes.py b/vis4d/zoo/base/datasets/nuscenes/nuscenes.py similarity index 100% rename from vis4d/config/common/datasets/nuscenes/nuscenes.py rename to vis4d/zoo/base/datasets/nuscenes/nuscenes.py diff --git a/vis4d/config/common/datasets/nuscenes/nuscenes_mono.py b/vis4d/zoo/base/datasets/nuscenes/nuscenes_mono.py similarity index 100% rename from vis4d/config/common/datasets/nuscenes/nuscenes_mono.py rename to vis4d/zoo/base/datasets/nuscenes/nuscenes_mono.py diff --git a/vis4d/config/common/datasets/shift/__init__.py b/vis4d/zoo/base/datasets/shift/__init__.py similarity index 100% rename from vis4d/config/common/datasets/shift/__init__.py rename to vis4d/zoo/base/datasets/shift/__init__.py diff --git a/vis4d/config/common/datasets/shift/common.py b/vis4d/zoo/base/datasets/shift/common.py similarity index 99% rename from vis4d/config/common/datasets/shift/common.py rename to vis4d/zoo/base/datasets/shift/common.py index 288dade1d..6d38d3031 100644 --- a/vis4d/config/common/datasets/shift/common.py +++ b/vis4d/zoo/base/datasets/shift/common.py @@ -7,12 +7,6 @@ from ml_collections.config_dict import ConfigDict from vis4d.config import class_config - -# from vis4d.config.default.dataloader import get_dataloader_config -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.shift import SHIFT @@ -47,6 +41,10 @@ ) from vis4d.data.transforms.select_sensor import SelectSensor from vis4d.data.transforms.to_tensor import ToTensor +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) IMAGE_MEAN = [122.884, 117.266, 110.287] IMAGE_STD = [59.925, 59.466, 60.69] diff --git a/vis4d/config/common/datasets/shift/tasks.py b/vis4d/zoo/base/datasets/shift/tasks.py similarity index 100% rename from vis4d/config/common/datasets/shift/tasks.py rename to vis4d/zoo/base/datasets/shift/tasks.py diff --git a/vis4d/zoo/base/models/__init__.py b/vis4d/zoo/base/models/__init__.py new file mode 100644 index 000000000..95d427c92 --- /dev/null +++ b/vis4d/zoo/base/models/__init__.py @@ -0,0 +1 @@ +"""Model Zoo base models.""" diff --git a/vis4d/config/common/models/faster_rcnn.py b/vis4d/zoo/base/models/faster_rcnn.py similarity index 100% rename from vis4d/config/common/models/faster_rcnn.py rename to vis4d/zoo/base/models/faster_rcnn.py diff --git a/vis4d/config/common/models/mask_rcnn.py b/vis4d/zoo/base/models/mask_rcnn.py similarity index 94% rename from vis4d/config/common/models/mask_rcnn.py rename to vis4d/zoo/base/models/mask_rcnn.py index 6781b940b..9f556e56d 100644 --- a/vis4d/config/common/models/mask_rcnn.py +++ b/vis4d/zoo/base/models/mask_rcnn.py @@ -5,7 +5,6 @@ from ml_collections import ConfigDict, FieldReference from vis4d.config import class_config -from vis4d.config.util import get_callable_cfg from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import ( LossConnector, @@ -27,10 +26,14 @@ ) from vis4d.op.detect.rcnn import RCNNHead, RCNNLoss from vis4d.op.detect.rpn import RPNLoss - -from .faster_rcnn import CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D -from .faster_rcnn import CONN_RPN_LOSS_2D as _CONN_RPN_LOSS_2D -from .faster_rcnn import ( +from vis4d.zoo.base import get_callable_cfg +from vis4d.zoo.base.models.faster_rcnn import ( + CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D, +) +from vis4d.zoo.base.models.faster_rcnn import ( + CONN_RPN_LOSS_2D as _CONN_RPN_LOSS_2D, +) +from vis4d.zoo.base.models.faster_rcnn import ( get_default_rcnn_box_codec_cfg, get_default_rpn_box_codec_cfg, ) diff --git a/vis4d/config/common/models/qdtrack.py b/vis4d/zoo/base/models/qdtrack.py similarity index 97% rename from vis4d/config/common/models/qdtrack.py rename to vis4d/zoo/base/models/qdtrack.py index e3671931a..ef7eca961 100644 --- a/vis4d/config/common/models/qdtrack.py +++ b/vis4d/zoo/base/models/qdtrack.py @@ -5,10 +5,6 @@ from ml_collections import ConfigDict, FieldReference from vis4d.config import class_config -from vis4d.config.common.models.faster_rcnn import ( - CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D, -) -from vis4d.config.util import get_callable_cfg from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import LossConnector, pred_key, remap_pred_keys from vis4d.engine.loss_module import LossModule @@ -26,11 +22,15 @@ QDTrackHead, QDTrackInstanceSimilarityLoss, ) - -from .faster_rcnn import ( +from vis4d.zoo.base import get_callable_cfg +from vis4d.zoo.base.models.faster_rcnn import ( + CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D, +) +from vis4d.zoo.base.models.faster_rcnn import ( get_default_rcnn_box_codec_cfg, get_default_rpn_box_codec_cfg, ) + from .yolox import get_yolox_model_cfg PRED_PREFIX = "detector_out" diff --git a/vis4d/config/common/models/yolox.py b/vis4d/zoo/base/models/yolox.py similarity index 98% rename from vis4d/config/common/models/yolox.py rename to vis4d/zoo/base/models/yolox.py index a5f5672cb..fa8946a97 100644 --- a/vis4d/config/common/models/yolox.py +++ b/vis4d/zoo/base/models/yolox.py @@ -7,8 +7,6 @@ from torch.optim.lr_scheduler import CosineAnnealingLR from vis4d.config import class_config -from vis4d.config.typing import OptimizerConfig -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.const import CommonKeys as K from vis4d.engine.callbacks import ( EMACallback, @@ -24,6 +22,8 @@ from vis4d.op.base import CSPDarknet from vis4d.op.detect.yolox import YOLOXHead, YOLOXHeadLoss from vis4d.op.fpp import YOLOXPAFPN +from vis4d.zoo.base import get_lr_scheduler_cfg, get_optimizer_cfg +from vis4d.zoo.typing import OptimizerConfig # Data connectors CONN_YOLOX_LOSS_2D = { diff --git a/vis4d/config/util/optimizer.py b/vis4d/zoo/base/optimizer.py similarity index 96% rename from vis4d/config/util/optimizer.py rename to vis4d/zoo/base/optimizer.py index 9d5bebcb5..3a23ed58c 100644 --- a/vis4d/config/util/optimizer.py +++ b/vis4d/zoo/base/optimizer.py @@ -4,11 +4,7 @@ from ml_collections import ConfigDict -from vis4d.config.typing import ( - LrSchedulerConfig, - OptimizerConfig, - ParamGroupCfg, -) +from vis4d.zoo.typing import LrSchedulerConfig, OptimizerConfig, ParamGroupCfg def get_lr_scheduler_cfg( diff --git a/vis4d/config/default/pl_trainer.py b/vis4d/zoo/base/pl_trainer.py similarity index 95% rename from vis4d/config/default/pl_trainer.py rename to vis4d/zoo/base/pl_trainer.py index e54e625cb..f05f979a6 100644 --- a/vis4d/config/default/pl_trainer.py +++ b/vis4d/zoo/base/pl_trainer.py @@ -5,7 +5,7 @@ from lightning import Trainer from vis4d.config import FieldConfigDict -from vis4d.config.typing import ExperimentConfig +from vis4d.zoo.typing import ExperimentConfig def get_default_pl_trainer_cfg(config: ExperimentConfig) -> ExperimentConfig: diff --git a/vis4d/config/default/runtime.py b/vis4d/zoo/base/runtime.py similarity index 96% rename from vis4d/config/default/runtime.py rename to vis4d/zoo/base/runtime.py index 111f78e75..1fba3dd45 100644 --- a/vis4d/config/default/runtime.py +++ b/vis4d/zoo/base/runtime.py @@ -8,8 +8,8 @@ from ml_collections import ConfigDict, FieldReference from vis4d.config import class_config -from vis4d.config.typing import ExperimentConfig from vis4d.engine.callbacks import CheckpointCallback, LoggingCallback +from vis4d.zoo.typing import ExperimentConfig def get_default_cfg( @@ -56,6 +56,7 @@ def get_default_cfg( config.seed = -1 config.log_every_n_steps = 50 config.use_tf32 = False + config.tf32_matmul_precision = "highest" config.benchmark = False return config diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py index 174a34ae0..310417cc4 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py @@ -7,29 +7,30 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_DET_EVAL, - get_bdd100k_detection_config, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_DET_EVAL, + get_bdd100k_detection_config, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py index ef415d9b2..0ef427dfa 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py @@ -7,29 +7,30 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_DET_EVAL, - get_bdd100k_detection_config, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_DET_EVAL, + get_bdd100k_detection_config, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py index 57c2d7c4e..472239c71 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py @@ -7,30 +7,31 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_INS_EVAL, - get_bdd100k_detection_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.const import CommonKeys as K +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.const import CommonKeys as K -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_INS_EVAL, + get_bdd100k_detection_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py index 0e061e38a..073f2bccb 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py @@ -7,30 +7,31 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_INS_EVAL, - get_bdd100k_detection_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.const import CommonKeys as K +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.const import CommonKeys as K -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_INS_EVAL, + get_bdd100k_detection_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py index 136973376..2845cf6c2 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py @@ -7,30 +7,31 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_INS_EVAL, - get_bdd100k_detection_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.const import CommonKeys as K +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.const import CommonKeys as K -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_INS_EVAL, + get_bdd100k_detection_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py b/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py index 6cf0d5cd1..7f4b79ae3 100644 --- a/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py @@ -7,28 +7,29 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( +from vis4d.data.datasets.bdd100k import bdd100k_track_map +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KTrackEvaluator +from vis4d.op.base import ResNet +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.datasets.bdd100k import ( CONN_BDD100K_TRACK_EVAL, get_bdd100k_track_cfg, ) -from vis4d.config.common.models.qdtrack import ( +from vis4d.zoo.base.models.qdtrack import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, get_qdtrack_cfg, ) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.datasets.bdd100k import bdd100k_track_map -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KTrackEvaluator -from vis4d.op.base import ResNet +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py index 2fbd87d10..277105196 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py @@ -7,23 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_SEG_EVAL, - get_bdd100k_sem_seg_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -38,6 +21,24 @@ from vis4d.op.base import ResNetV1c from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_SEG_EVAL, + get_bdd100k_sem_seg_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py index b1a52ae89..b1a3ede4f 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py @@ -7,23 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_SEG_EVAL, - get_bdd100k_sem_seg_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -37,6 +20,24 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_SEG_EVAL, + get_bdd100k_sem_seg_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py index 7fd4b9817..cd0657845 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py @@ -7,23 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import ( - CONN_BDD100K_SEG_EVAL, - get_bdd100k_sem_seg_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -37,6 +20,24 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.bdd100k import ( + CONN_BDD100K_SEG_EVAL, + get_bdd100k_sem_seg_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/bevformer_base.py b/vis4d/zoo/bevformer/bevformer_base.py index a930550ba..28c22c599 100644 --- a/vis4d/zoo/bevformer/bevformer_base.py +++ b/vis4d/zoo/bevformer/bevformer_base.py @@ -7,25 +7,26 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import CallbackConnector, MultiSensorDataConnector from vis4d.eval.nuscenes import NuScenesDet3DEvaluator from vis4d.model.detect3d.bevformer import BEVFormer from vis4d.op.base import ResNet +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) from vis4d.zoo.bevformer.data import ( CONN_NUSC_BBOX_3D_TEST, CONN_NUSC_DET3D_EVAL, get_nusc_cfg, nuscenes_class_map, ) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/bevformer_tiny.py b/vis4d/zoo/bevformer/bevformer_tiny.py index f5420426e..289330e2e 100644 --- a/vis4d/zoo/bevformer/bevformer_tiny.py +++ b/vis4d/zoo/bevformer/bevformer_tiny.py @@ -7,13 +7,6 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import CallbackConnector, MultiSensorDataConnector @@ -31,12 +24,20 @@ ) from vis4d.op.detect3d.bevformer.transformer import PerceptionTransformer from vis4d.op.fpp.fpn import FPN +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) from vis4d.zoo.bevformer.data import ( CONN_NUSC_BBOX_3D_TEST, CONN_NUSC_DET3D_EVAL, get_nusc_cfg, nuscenes_class_map, ) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: @@ -48,7 +49,7 @@ def get_config() -> ExperimentConfig: ###################################################### ## General Config ## ###################################################### - config = get_default_cfg(exp_name="bevformer_base") + config = get_default_cfg(exp_name="bevformer_tiny") # Hyper Parameters params = ExperimentParameters() diff --git a/vis4d/zoo/bevformer/bevformer_vis.py b/vis4d/zoo/bevformer/bevformer_vis.py index d762235c4..c75b30a11 100644 --- a/vis4d/zoo/bevformer/bevformer_vis.py +++ b/vis4d/zoo/bevformer/bevformer_vis.py @@ -3,11 +3,10 @@ from __future__ import annotations from vis4d.config import class_config -from vis4d.config.default import get_default_callbacks_cfg -from vis4d.config.typing import ExperimentConfig from vis4d.engine.callbacks import VisualizerCallback from vis4d.engine.connectors import MultiSensorCallbackConnector from vis4d.vis.image.bbox3d_visualizer import MultiCameraBBox3DVisualizer +from vis4d.zoo.base import get_default_callbacks_cfg from vis4d.zoo.bevformer.bevformer_base import ( get_config as get_bevformer_config, ) @@ -16,6 +15,7 @@ NUSC_CAMERAS, nuscenes_class_map, ) +from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/data.py b/vis4d/zoo/bevformer/data.py index 79651bd38..937ef46ba 100644 --- a/vis4d/zoo/bevformer/data.py +++ b/vis4d/zoo/bevformer/data.py @@ -5,12 +5,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.common.datasets.nuscenes import ( - get_nusc_mini_val_cfg, - get_nusc_val_cfg, -) -from vis4d.config.typing import DataConfig -from vis4d.config.util import get_inference_dataloaders_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.loader import multi_sensor_collate @@ -24,6 +18,12 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import get_inference_dataloaders_cfg +from vis4d.zoo.base.datasets.nuscenes import ( + get_nusc_mini_val_cfg, + get_nusc_val_cfg, +) +from vis4d.zoo.typing import DataConfig nuscenes_class_map = { "car": 0, diff --git a/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py index ddf9740fc..6c93f4576 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py @@ -3,7 +3,6 @@ from __future__ import annotations from vis4d.config import class_config -from vis4d.config.typing import DataConfig, ExperimentConfig from vis4d.data.const import CommonKeys as K from vis4d.data.datasets.nuscenes import NuScenes from vis4d.data.datasets.nuscenes_detection import NuScenesDetection @@ -18,6 +17,7 @@ get_config as get_velo_lstm_cfg, ) from vis4d.zoo.cc_3dt.data import CONN_NUSC_BBOX_3D_TEST, get_test_dataloader +from vis4d.zoo.typing import DataConfig, ExperimentConfig CONN_NUSC_BBOX_3D_TEST = { "images_list": data_key(K.images, sensors=NuScenes.CAMERAS), diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py index 99fadbefd..61efb82e5 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py @@ -7,13 +7,6 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -27,6 +20,13 @@ NuScenesTrack3DEvaluator, ) from vis4d.op.base import ResNet +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) from vis4d.zoo.cc_3dt.data import ( CONN_NUSC_BBOX_3D_TEST, CONN_NUSC_DET3D_EVAL, @@ -34,6 +34,7 @@ get_nusc_cfg, ) from vis4d.zoo.cc_3dt.model import CONN_BBOX_3D_TRAIN, get_cc_3dt_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py index 34d1033b9..c2e047034 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py @@ -3,19 +3,19 @@ from __future__ import annotations from vis4d.config import class_config -from vis4d.config.default import get_default_callbacks_cfg -from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import NuScenes, nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import MultiSensorCallbackConnector from vis4d.eval.nuscenes import NuScenesDet3DEvaluator from vis4d.op.base import ResNet +from vis4d.zoo.base import get_default_callbacks_cfg from vis4d.zoo.cc_3dt.cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc import ( get_config as get_kf3d_config, ) from vis4d.zoo.cc_3dt.data import CONN_NUSC_DET3D_EVAL, get_nusc_cfg from vis4d.zoo.cc_3dt.model import get_cc_3dt_cfg +from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py index 343ede71b..5027cea66 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py @@ -3,7 +3,6 @@ from __future__ import annotations from vis4d.config import class_config -from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.model.motion.velo_lstm import VeloLSTM from vis4d.op.base import ResNet @@ -11,6 +10,7 @@ get_config as get_kf3d_cfg, ) from vis4d.zoo.cc_3dt.model import get_cc_3dt_cfg +from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py index 5fe10a43a..d9e289039 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py @@ -7,13 +7,6 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -27,6 +20,13 @@ NuScenesTrack3DEvaluator, ) from vis4d.op.base import ResNet +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) from vis4d.zoo.cc_3dt.data import ( CONN_NUSC_BBOX_3D_TEST, CONN_NUSC_DET3D_EVAL, @@ -34,6 +34,7 @@ get_nusc_cfg, ) from vis4d.zoo.cc_3dt.model import CONN_BBOX_3D_TRAIN, get_cc_3dt_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py b/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py index 4867f7240..5a0b8428a 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py @@ -3,13 +3,12 @@ from __future__ import annotations from vis4d.config import class_config -from vis4d.config.default import get_default_callbacks_cfg -from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import NuScenes, nuscenes_class_map from vis4d.engine.callbacks import VisualizerCallback from vis4d.engine.connectors import MultiSensorCallbackConnector from vis4d.vis.image.bbox3d_visualizer import MultiCameraBBox3DVisualizer from vis4d.vis.image.bev_visualizer import BEVBBox3DVisualizer +from vis4d.zoo.base import get_default_callbacks_cfg from vis4d.zoo.cc_3dt.cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc import ( get_config as get_cc_3dt_config, ) @@ -17,6 +16,7 @@ CONN_NUSC_BBOX_3D_VIS, CONN_NUSC_BEV_BBOX_3D_VIS, ) +from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/data.py b/vis4d/zoo/cc_3dt/data.py index e42a29330..7cae8b4ad 100644 --- a/vis4d/zoo/cc_3dt/data.py +++ b/vis4d/zoo/cc_3dt/data.py @@ -5,18 +5,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.common.datasets.nuscenes import ( - get_nusc_mini_val_cfg, - get_nusc_mono_mini_train_cfg, - get_nusc_mono_train_cfg, - get_nusc_train_cfg, - get_nusc_val_cfg, -) -from vis4d.config.typing import DataConfig -from vis4d.config.util import ( - get_inference_dataloaders_cfg, - get_train_dataloader_cfg, -) from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.nuscenes import NuScenes @@ -40,6 +28,18 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import ( + get_inference_dataloaders_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.base.datasets.nuscenes import ( + get_nusc_mini_val_cfg, + get_nusc_mono_mini_train_cfg, + get_nusc_mono_train_cfg, + get_nusc_train_cfg, + get_nusc_val_cfg, +) +from vis4d.zoo.typing import DataConfig CONN_NUSC_DET3D_EVAL = { "tokens": data_key("token"), diff --git a/vis4d/zoo/cc_3dt/model.py b/vis4d/zoo/cc_3dt/model.py index 571927c16..39d6c9b19 100644 --- a/vis4d/zoo/cc_3dt/model.py +++ b/vis4d/zoo/cc_3dt/model.py @@ -5,17 +5,6 @@ from ml_collections import ConfigDict, FieldReference from vis4d.config import class_config -from vis4d.config.common.models.faster_rcnn import ( - get_default_rcnn_box_codec_cfg, - get_default_rpn_box_codec_cfg, -) -from vis4d.config.common.models.qdtrack import ( - CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D, -) -from vis4d.config.common.models.qdtrack import ( - CONN_TRACK_LOSS_2D as _CONN_TRACK_LOSS_2D, -) -from vis4d.config.util import get_callable_cfg from vis4d.data.const import CommonKeys as K from vis4d.engine.connectors import LossConnector, pred_key, remap_pred_keys from vis4d.engine.loss_module import LossModule @@ -28,6 +17,15 @@ from vis4d.op.loss.common import smooth_l1_loss from vis4d.op.track.qdtrack import QDTrackInstanceSimilarityLoss from vis4d.state.track3d.cc_3dt import CC3DTrackGraph +from vis4d.zoo.base import get_callable_cfg +from vis4d.zoo.base.models.faster_rcnn import ( + get_default_rcnn_box_codec_cfg, + get_default_rpn_box_codec_cfg, +) +from vis4d.zoo.base.models.qdtrack import CONN_ROI_LOSS_2D as _CONN_ROI_LOSS_2D +from vis4d.zoo.base.models.qdtrack import ( + CONN_TRACK_LOSS_2D as _CONN_TRACK_LOSS_2D, +) PRED_PREFIX = "qdtrack_out" diff --git a/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py b/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py index 798c1901e..cc9be21f2 100644 --- a/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py +++ b/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import MultiStepLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ( - DataConfig, - ExperimentConfig, - ExperimentParameters, -) -from vis4d.config.util import ( - get_lr_scheduler_cfg, - get_optimizer_cfg, - get_train_dataloader_cfg, -) from vis4d.data.datasets.nuscenes_trajectory import NuScenesTrajectory from vis4d.engine.connectors import ( DataConnector, @@ -32,6 +17,15 @@ from vis4d.engine.loss_module import LossModule from vis4d.model.motion.velo_lstm import VeloLSTM from vis4d.op.motion.velo_lstm import VeloLSTMLoss +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig, ExperimentConfig, ExperimentParameters TRAJ_TRAIN = {"pred_traj": "pred_traj"} TRAJ_LOSS = { diff --git a/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py b/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py index 75360d31f..43b1d0f98 100644 --- a/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py +++ b/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import MultiStepLR from vis4d.config import class_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.typing import ( - DataConfig, - ExperimentConfig, - ExperimentParameters, -) -from vis4d.config.util import ( - get_lr_scheduler_cfg, - get_optimizer_cfg, - get_train_dataloader_cfg, -) from vis4d.data.datasets.nuscenes_trajectory import NuScenesTrajectory from vis4d.engine.connectors import ( DataConnector, @@ -32,6 +17,15 @@ from vis4d.engine.loss_module import LossModule from vis4d.model.motion.velo_lstm import VeloLSTM from vis4d.op.motion.velo_lstm import VeloLSTMLoss +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, + get_train_dataloader_cfg, +) +from vis4d.zoo.typing import DataConfig, ExperimentConfig, ExperimentParameters TRAJ_TRAIN = {"pred_traj": "pred_traj"} TRAJ_LOSS = { diff --git a/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py b/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py index 984035220..e3a166ff9 100644 --- a/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py +++ b/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py @@ -8,34 +8,35 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.coco import ( - CONN_COCO_BBOX_EVAL, - get_coco_detection_cfg, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.config.sweep import grid_search +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.coco import COCODetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ( +from vis4d.zoo.base.datasets.coco import ( + CONN_COCO_BBOX_EVAL, + get_coco_detection_cfg, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ( ExperimentConfig, ExperimentParameters, ParameterSweepConfig, ) -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.config.util.sweep import grid_search -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.coco import COCODetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py b/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py index 4a987b907..2c41c8ce5 100644 --- a/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py +++ b/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py @@ -7,25 +7,26 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.coco import get_coco_sem_seg_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.connectors import DataConnector, LossConnector +from vis4d.engine.loss_module import LossModule +from vis4d.engine.optim import PolyLR +from vis4d.model.seg.fcn_resnet import FCNResNet +from vis4d.op.loss import MultiLevelSegLoss +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors.seg import ( +from vis4d.zoo.base.data_connectors.seg import ( CONN_MASKS_TEST, CONN_MASKS_TRAIN, CONN_MULTI_SEG_LOSS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.connectors import DataConnector, LossConnector -from vis4d.engine.loss_module import LossModule -from vis4d.engine.optim import PolyLR -from vis4d.model.seg.fcn_resnet import FCNResNet -from vis4d.op.loss import MultiLevelSegLoss +from vis4d.zoo.base.datasets.coco import get_coco_sem_seg_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py index d00f88076..7379cc866 100644 --- a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py +++ b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py @@ -7,23 +7,6 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.coco import ( - CONN_COCO_BBOX_EVAL, - get_coco_detection_cfg, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import ( - CONN_BBOX_2D_TEST, - CONN_BBOX_2D_TRAIN, - CONN_BBOX_2D_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -35,6 +18,24 @@ from vis4d.eval.coco import COCODetectEvaluator from vis4d.op.base import ResNet from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors import ( + CONN_BBOX_2D_TEST, + CONN_BBOX_2D_TRAIN, + CONN_BBOX_2D_VIS, +) +from vis4d.zoo.base.datasets.coco import ( + CONN_COCO_BBOX_EVAL, + get_coco_detection_cfg, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/qdtrack/data_yolox.py b/vis4d/zoo/qdtrack/data_yolox.py index 26c96946c..9ce8b9bb7 100644 --- a/vis4d/zoo/qdtrack/data_yolox.py +++ b/vis4d/zoo/qdtrack/data_yolox.py @@ -5,9 +5,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import get_inference_dataloaders_cfg -from vis4d.config.util.callable import get_callable_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe, MultiSampleDataPipe from vis4d.data.datasets.bdd100k import BDD100K, bdd100k_track_map @@ -48,6 +45,9 @@ ResizeImages, ) from vis4d.data.transforms.to_tensor import ToTensor +from vis4d.zoo.base import get_inference_dataloaders_cfg +from vis4d.zoo.base.callable import get_callable_cfg +from vis4d.zoo.typing import DataConfig def get_train_dataloader( diff --git a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py index 1cc9926ad..05322b2e5 100644 --- a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py @@ -7,20 +7,6 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL -from vis4d.config.common.models.qdtrack import ( - CONN_BBOX_2D_TEST, - CONN_BBOX_2D_TRAIN, - get_qdtrack_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import CONN_BBOX_2D_TRACK_VIS -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.datasets.bdd100k import bdd100k_track_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import ( @@ -32,7 +18,22 @@ from vis4d.eval.bdd100k import BDD100KTrackEvaluator from vis4d.op.base import ResNet from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TRACK_VIS +from vis4d.zoo.base.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL +from vis4d.zoo.base.models.qdtrack import ( + CONN_BBOX_2D_TEST, + CONN_BBOX_2D_TRAIN, + get_qdtrack_cfg, +) from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py index 5c7cb3b09..e19f82562 100644 --- a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py @@ -6,30 +6,30 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config -from vis4d.config.common.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL -from vis4d.config.common.models.qdtrack import ( +from vis4d.data.datasets.bdd100k import bdd100k_track_map +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KTrackEvaluator +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, +) +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TRACK_VIS +from vis4d.zoo.base.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL +from vis4d.zoo.base.models.qdtrack import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, get_qdtrack_yolox_cfg, ) -from vis4d.config.common.models.yolox import ( +from vis4d.zoo.base.models.yolox import ( get_yolox_callbacks_cfg, get_yolox_optimizers_cfg, ) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import CONN_BBOX_2D_TRACK_VIS -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.data.datasets.bdd100k import bdd100k_track_map -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KTrackEvaluator -from vis4d.vis.image import BoundingBoxVisualizer from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/retinanet/retinanet_coco.py b/vis4d/zoo/retinanet/retinanet_coco.py index 9f6e42217..b4974e157 100644 --- a/vis4d/zoo/retinanet/retinanet_coco.py +++ b/vis4d/zoo/retinanet/retinanet_coco.py @@ -7,23 +7,6 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.coco import ( - CONN_COCO_BBOX_EVAL, - get_coco_detection_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import ( - CONN_BBOX_2D_VIS, - CONN_BOX_LOSS_2D, - CONN_IMAGES_TEST, - CONN_IMAGES_TRAIN, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -40,6 +23,24 @@ get_default_anchor_generator, ) from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors import ( + CONN_BBOX_2D_VIS, + CONN_BOX_LOSS_2D, + CONN_IMAGES_TEST, + CONN_IMAGES_TRAIN, +) +from vis4d.zoo.base.datasets.coco import ( + CONN_COCO_BBOX_EVAL, + get_coco_detection_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py index 46e7ca27e..e533d7bb5 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py @@ -7,29 +7,30 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_DET_EVAL, - get_shift_det_config, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_DET_EVAL, + get_shift_det_config, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py index 1abd1b5cc..54bc2314b 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py @@ -7,29 +7,30 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_DET_EVAL, - get_shift_det_config, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_DET_EVAL, + get_shift_det_config, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py index c56a12b8b..6b549994b 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py @@ -7,29 +7,30 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_DET_EVAL, - get_shift_det_config, -) -from vis4d.config.common.models import get_faster_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_BBOX_2D_VIS, ) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_DET_EVAL, + get_shift_det_config, +) +from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py index 7930d802e..d9da7bb43 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py @@ -7,28 +7,29 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import FieldConfigDict, class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_INS_EVAL, - get_shift_instance_seg_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_INS_EVAL, + get_shift_instance_seg_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg def get_config() -> FieldConfigDict: diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py index c63ec3d0e..40963714e 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py @@ -7,28 +7,29 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import FieldConfigDict, class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_INS_EVAL, - get_shift_instance_seg_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_INS_EVAL, + get_shift_instance_seg_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg def get_config() -> FieldConfigDict: diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py index b243c45b5..1f915339d 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py @@ -7,28 +7,29 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import FieldConfigDict, class_config -from vis4d.config.common.datasets.shift import ( - CONN_SHIFT_INS_EVAL, - get_shift_instance_seg_config, -) -from vis4d.config.common.models import get_mask_rcnn_cfg -from vis4d.config.default import ( +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.shift import SHIFTDetectEvaluator +from vis4d.op.base import ResNet +from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( get_default_callbacks_cfg, get_default_cfg, get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, ) -from vis4d.config.default.data_connectors import ( +from vis4d.zoo.base.data_connectors import ( CONN_BBOX_2D_TEST, CONN_BBOX_2D_TRAIN, CONN_INS_MASK_2D_VIS, ) -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.shift import SHIFTDetectEvaluator -from vis4d.op.base import ResNet -from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base.datasets.shift import ( + CONN_SHIFT_INS_EVAL, + get_shift_instance_seg_config, +) +from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg def get_config() -> FieldConfigDict: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py index 378ec14fd..aa330afb4 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import get_shift_sem_seg_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_EVAL, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,6 +20,22 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_EVAL, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py index dc205648f..b76fdbeae 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import get_shift_sem_seg_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_EVAL, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,6 +20,22 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_EVAL, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py index 801a5dbfc..29cf9d960 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import get_shift_sem_seg_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_EVAL, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,6 +20,22 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_EVAL, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py index 178f4c6dc..d9609413d 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py @@ -7,21 +7,6 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.shift import get_shift_sem_seg_config -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.seg import ( - CONN_MASKS_TEST, - CONN_MASKS_TRAIN, - CONN_SEG_EVAL, - CONN_SEG_LOSS, - CONN_SEG_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,6 +20,22 @@ from vis4d.model.seg.semantic_fpn import SemanticFPN from vis4d.op.loss import SegCrossEntropyLoss from vis4d.vis.image import SegMaskVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.seg import ( + CONN_MASKS_TEST, + CONN_MASKS_TRAIN, + CONN_SEG_EVAL, + CONN_SEG_LOSS, + CONN_SEG_VIS, +) +from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/config/typing.py b/vis4d/zoo/typing.py similarity index 98% rename from vis4d/config/typing.py rename to vis4d/zoo/typing.py index b6b49b240..9c8ff8f3a 100644 --- a/vis4d/config/typing.py +++ b/vis4d/zoo/typing.py @@ -41,7 +41,7 @@ class DataConfig(ConfigDict): # type: ignore Example: >>> from vis4d.config.types import DataConfig - >>> from vis4d.config.util import class_config + >>> from vis4d.zoo.base import class_config >>> from my_package.data import MyDataLoader >>> cfg = DataConfig() >>> cfg.train_dataloader = class_config(MyDataLoader, ...) @@ -148,6 +148,7 @@ class ExperimentConfig(FieldConfigDict): log_every_n_steps: int | FieldReference use_tf32: bool | FieldReference benchmark: bool | FieldReference + tf32_matmul_precision: str | FieldReference params: ExperimentParameters diff --git a/vis4d/zoo/vit/vit_small_imagenet.py b/vis4d/zoo/vit/vit_small_imagenet.py index 92412f177..b19a3608d 100644 --- a/vis4d/zoo/vit/vit_small_imagenet.py +++ b/vis4d/zoo/vit/vit_small_imagenet.py @@ -8,22 +8,6 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.imagenet import ( - CONN_IMAGENET_CLS_EVAL, - get_imagenet_cls_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.cls import ( - CONN_CLS_LOSS, - CONN_CLS_TEST, - CONN_CLS_TRAIN, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.engine.callbacks import EMACallback, EvaluatorCallback from vis4d.engine.connectors import ( CallbackConnector, @@ -34,6 +18,23 @@ from vis4d.eval.common.cls import ClassificationEvaluator from vis4d.model.adapter import ModelEMAAdapter from vis4d.model.cls.vit import ViTClassifer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.cls import ( + CONN_CLS_LOSS, + CONN_CLS_TEST, + CONN_CLS_TRAIN, +) +from vis4d.zoo.base.datasets.imagenet import ( + CONN_IMAGENET_CLS_EVAL, + get_imagenet_cls_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/vit/vit_tiny_imagenet.py b/vis4d/zoo/vit/vit_tiny_imagenet.py index bc7a601ca..ada082059 100644 --- a/vis4d/zoo/vit/vit_tiny_imagenet.py +++ b/vis4d/zoo/vit/vit_tiny_imagenet.py @@ -8,22 +8,6 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config -from vis4d.config.common.datasets.imagenet import ( - CONN_IMAGENET_CLS_EVAL, - get_imagenet_cls_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors.cls import ( - CONN_CLS_LOSS, - CONN_CLS_TEST, - CONN_CLS_TRAIN, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.config.util import get_lr_scheduler_cfg, get_optimizer_cfg from vis4d.engine.callbacks import EMACallback, EvaluatorCallback from vis4d.engine.connectors import ( CallbackConnector, @@ -34,6 +18,23 @@ from vis4d.eval.common.cls import ClassificationEvaluator from vis4d.model.adapter import ModelEMAAdapter from vis4d.model.cls.vit import ViTClassifer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, + get_lr_scheduler_cfg, + get_optimizer_cfg, +) +from vis4d.zoo.base.data_connectors.cls import ( + CONN_CLS_LOSS, + CONN_CLS_TEST, + CONN_CLS_TRAIN, +) +from vis4d.zoo.base.datasets.imagenet import ( + CONN_IMAGENET_CLS_EVAL, + get_imagenet_cls_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/yolox/data.py b/vis4d/zoo/yolox/data.py index 3739e4a62..dde2e30cd 100644 --- a/vis4d/zoo/yolox/data.py +++ b/vis4d/zoo/yolox/data.py @@ -7,9 +7,6 @@ from ml_collections import ConfigDict from vis4d.config import class_config -from vis4d.config.typing import DataConfig -from vis4d.config.util import get_inference_dataloaders_cfg -from vis4d.config.util.callable import get_callable_cfg from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe, MultiSampleDataPipe from vis4d.data.datasets.coco import COCO @@ -42,6 +39,9 @@ ) from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key +from vis4d.zoo.base import get_inference_dataloaders_cfg +from vis4d.zoo.base.callable import get_callable_cfg +from vis4d.zoo.typing import DataConfig CONN_COCO_BBOX_EVAL = { "coco_image_id": data_key(K.sample_names), diff --git a/vis4d/zoo/yolox/yolox_s_300e_coco.py b/vis4d/zoo/yolox/yolox_s_300e_coco.py index aedd6ec2a..c47a49171 100644 --- a/vis4d/zoo/yolox/yolox_s_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_s_300e_coco.py @@ -6,27 +6,24 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config -from vis4d.config.common.models.yolox import ( - get_yolox_callbacks_cfg, - get_yolox_cfg, - get_yolox_optimizers_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import ( - CONN_BBOX_2D_TEST, - CONN_BBOX_2D_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector from vis4d.eval.coco import COCODetectEvaluator from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, +) +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TEST, CONN_BBOX_2D_VIS +from vis4d.zoo.base.models.yolox import ( + get_yolox_callbacks_cfg, + get_yolox_cfg, + get_yolox_optimizers_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters from vis4d.zoo.yolox.data import CONN_COCO_BBOX_EVAL, get_coco_yolox_cfg CONN_BBOX_2D_TRAIN = {"images": K.images} diff --git a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py index 09a55cd4c..83dcc36f0 100644 --- a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py @@ -6,27 +6,24 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config -from vis4d.config.common.models.yolox import ( - get_yolox_callbacks_cfg, - get_yolox_cfg, - get_yolox_optimizers_cfg, -) -from vis4d.config.default import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.config.default.data_connectors import ( - CONN_BBOX_2D_TEST, - CONN_BBOX_2D_VIS, -) -from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector from vis4d.eval.coco import COCODetectEvaluator from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, +) +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TEST, CONN_BBOX_2D_VIS +from vis4d.zoo.base.models.yolox import ( + get_yolox_callbacks_cfg, + get_yolox_cfg, + get_yolox_optimizers_cfg, +) +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters from vis4d.zoo.yolox.data import CONN_COCO_BBOX_EVAL, get_coco_yolox_cfg CONN_BBOX_2D_TRAIN = {"images": K.images} From f07bddb2a20db29c5fe0a69f7e3ebff025489f87 Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Thu, 11 Apr 2024 17:57:50 +0200 Subject: [PATCH 02/12] feat: Update test data. --- tests/vis4d-test-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index a627bd3e6..12ae687e3 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit a627bd3e69bf8961809c365eef65ac874fb030b8 +Subproject commit 12ae687e3b468de14501867de85a1cb61df4e520 From ad44d4bb052d43734c01f829919b1631dc363ebe Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Thu, 11 Apr 2024 18:20:31 +0200 Subject: [PATCH 03/12] feat: Support submission for nuscenes. --- vis4d/data/datasets/nuscenes.py | 41 +++++----- vis4d/op/track3d/cc_3dt.py | 2 +- vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py | 110 +++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py diff --git a/vis4d/data/datasets/nuscenes.py b/vis4d/data/datasets/nuscenes.py index a9b224b29..71d7edfa7 100644 --- a/vis4d/data/datasets/nuscenes.py +++ b/vis4d/data/datasets/nuscenes.py @@ -240,6 +240,9 @@ def get_cat_ids(self, idx: int) -> list[int]: def _filter_data(self, data: list[DictStrAny]) -> list[DictStrAny]: """Remove empty samples.""" + if self.split == "test": + return data + samples = [] frequencies = {cat: 0 for cat in nuscenes_class_map} inv_nuscenes_class_map = {v: k for k, v in nuscenes_class_map.items()} @@ -424,13 +427,15 @@ def _generate_data_mapping(self) -> list[DictStrAny]: lidar_token = sample["data"]["LIDAR_TOP"] frame["LIDAR_TOP"] = self._load_lidar_data(data, lidar_token) - frame["LIDAR_TOP"]["annotations"] = self._load_annotations( - data, - frame["LIDAR_TOP"]["extrinsics"], - sample["anns"], - instance_tokens, - axis_mode=AxisMode.LIDAR, - ) + + if self.split != "test": + frame["LIDAR_TOP"]["annotations"] = self._load_annotations( + data, + frame["LIDAR_TOP"]["extrinsics"], + sample["anns"], + instance_tokens, + axis_mode=AxisMode.LIDAR, + ) # obtain sweeps for a single key-frame sweeps: list[DictStrAny] = [] @@ -448,16 +453,18 @@ def _generate_data_mapping(self) -> list[DictStrAny]: cam_token = sample["data"][cam] frame[cam] = self._load_cam_data(data, cam_token) - frame[cam]["annotations"] = self._load_annotations( - data, - frame[cam]["extrinsics"], - sample["anns"], - instance_tokens, - axis_mode=AxisMode.OPENCV, - export_2d_annotations=True, - intrinsics=frame[cam]["intrinsics"], - image_hw=frame[cam]["image_hw"], - ) + + if self.split != "test": + frame[cam]["annotations"] = self._load_annotations( + data, + frame[cam]["extrinsics"], + sample["anns"], + instance_tokens, + axis_mode=AxisMode.OPENCV, + export_2d_annotations=True, + intrinsics=frame[cam]["intrinsics"], + image_hw=frame[cam]["image_hw"], + ) # TODO add RADAR, Map diff --git a/vis4d/op/track3d/cc_3dt.py b/vis4d/op/track3d/cc_3dt.py index 2838e8b8f..b7caa87c4 100644 --- a/vis4d/op/track3d/cc_3dt.py +++ b/vis4d/op/track3d/cc_3dt.py @@ -324,8 +324,8 @@ def __call__( # Depth Ordering scores_depth = self.depth_ordering( detections_3d, - memory_boxes_3d, memory_boxes_3d_predict, + memory_boxes_3d, memory_velocities, ) diff --git a/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py new file mode 100644 index 000000000..fa7b8966a --- /dev/null +++ b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py @@ -0,0 +1,110 @@ +# pylint: disable=duplicate-code +"""CC-3DT with BEV detector on nuScenes.""" +from __future__ import annotations + +from vis4d.config import class_config +from vis4d.data.const import CommonKeys as K +from vis4d.data.datasets.nuscenes import NuScenes +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback +from vis4d.engine.connectors import ( + CallbackConnector, + MultiSensorCallbackConnector, +) +from vis4d.eval.nuscenes import ( + NuScenesDet3DEvaluator, + NuScenesTrack3DEvaluator, +) +from vis4d.zoo.base import get_default_callbacks_cfg +from vis4d.zoo.cc_3dt.cc_3dt_bevformer_base_velo_lstm_nusc import ( + get_config as get_cc_3dt_config, +) +from vis4d.zoo.cc_3dt.data import ( + CONN_NUSC_DET3D_EVAL, + CONN_NUSC_TRACK3D_EVAL, + get_test_dataloader, +) +from vis4d.zoo.typing import DataConfig, ExperimentConfig + + +def get_config() -> ExperimentConfig: + """Returns the config dict for CC-3DT on nuScenes. + + Returns: + ExperimentConfig: The configuration + """ + ###################################################### + ## General Config ## + ###################################################### + config = get_cc_3dt_config().ref_mode() + + config.experiment_name = "cc_3dt_nusc_test" + + ###################################################### + ## Datasets with augmentations ## + ###################################################### + config.pure_detection = "" + + data = DataConfig() + + data.train_dataloader = None + + test_dataset = class_config( + NuScenes, + data_root="data/nuscenes", + version="v1.0-test", + split="test", + keys_to_load=[K.images, K.original_images], + data_backend=class_config(HDF5Backend), + cache_as_binary=True, + cached_file_path="data/nuscenes/test.pkl", + ) + + data.test_dataloader = get_test_dataloader( + test_dataset=test_dataset, samples_per_gpu=1, workers_per_gpu=4 + ) + + config.data = data + + ###################################################### + ## CALLBACKS ## + ###################################################### + # Logger and Checkpoint + callbacks = get_default_callbacks_cfg(config.output_dir) + + # Evaluator + callbacks.append( + class_config( + EvaluatorCallback, + evaluator=class_config( + NuScenesDet3DEvaluator, + data_root="data/nuscenes", + version="v1.0-test", + split="test", + save_only=True, + ), + save_predictions=True, + save_prefix=config.output_dir, + test_connector=class_config( + MultiSensorCallbackConnector, + key_mapping=CONN_NUSC_DET3D_EVAL, + sensors=NuScenes.CAMERAS, + ), + ) + ) + + callbacks.append( + class_config( + EvaluatorCallback, + evaluator=class_config(NuScenesTrack3DEvaluator), + save_predictions=True, + save_prefix=config.output_dir, + test_connector=class_config( + CallbackConnector, key_mapping=CONN_NUSC_TRACK3D_EVAL + ), + ) + ) + + config.callbacks = callbacks + + return config.value_mode() From a39d8db2a419bc606e71b181602fb630516b7c23 Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Fri, 19 Apr 2024 18:53:22 +0200 Subject: [PATCH 04/12] feat: Update config typing; refactor ResNet, transformer and CC-3DT related. --- docs/source/user_guide/faster_rcnn_example.py | 10 +- pyproject.toml | 1 + tests/config/sweep_test.py | 2 +- tests/engine/optim/optimizer_test.py | 2 +- tests/eval/common/depth_test.py | 2 +- tests/eval/kitti/depth_test.py | 2 +- tests/op/base/resnet_test.py | 4 +- tests/vis4d-test-data | 2 +- tests/zoo/util.py | 2 +- vis4d/common/imports.py | 1 + vis4d/common/logging.py | 12 +++ vis4d/common/typing.py | 22 ++-- vis4d/{zoo => config}/typing.py | 2 +- vis4d/data/const.py | 13 ++- vis4d/data/datasets/nuscenes.py | 6 +- vis4d/data/transforms/affine.py | 18 +--- vis4d/data/transforms/crop.py | 101 +++++++++++------- vis4d/data/transforms/mask.py | 6 +- vis4d/data/transforms/mixup.py | 12 +-- vis4d/data/transforms/mosaic.py | 10 +- vis4d/data/transforms/photometric.py | 2 +- vis4d/data/transforms/post_process.py | 42 ++++---- vis4d/data/transforms/resize.py | 22 +++- vis4d/data/transforms/to_tensor.py | 12 ++- vis4d/engine/callbacks/evaluator.py | 17 +-- vis4d/engine/experiment.py | 9 +- vis4d/engine/optim/optimizer.py | 2 +- vis4d/engine/optim/scheduler.py | 17 +-- vis4d/engine/run.py | 2 +- vis4d/eval/common/depth.py | 12 ++- vis4d/eval/metrics/depth.py | 2 +- vis4d/model/adapter/flops.py | 59 ++++++++++ vis4d/model/seg/semantic_fpn.py | 4 +- vis4d/model/segment3d/pointnetpp.py | 51 ++------- vis4d/op/base/resnet.py | 101 ++++++++---------- vis4d/op/box/box2d.py | 26 ++++- vis4d/op/detect/rcnn.py | 7 +- vis4d/op/detect3d/bevformer/decoder.py | 17 ++- vis4d/op/detect3d/bevformer/encoder.py | 22 ++-- vis4d/op/detect3d/qd_3dt.py | 2 - vis4d/op/layer/attention.py | 3 +- vis4d/op/layer/ms_deform_attn.py | 1 + vis4d/op/layer/transformer.py | 45 ++++++-- vis4d/op/layer/util.py | 48 ++++++++- vis4d/op/loss/cross_entropy.py | 89 +++++++++++++++ vis4d/op/loss/multi_level_seg_loss.py | 27 ++--- vis4d/op/loss/seg_cross_entropy_loss.py | 39 +++---- vis4d/op/mask/util.py | 23 ++++ vis4d/op/seg/semantic_fpn.py | 13 ++- vis4d/pl/data_module.py | 2 +- vis4d/pl/run.py | 12 ++- vis4d/pl/training_module.py | 31 +++++- vis4d/vis/image/util.py | 4 +- vis4d/zoo/base/datasets/bdd100k/detect.py | 2 +- vis4d/zoo/base/datasets/bdd100k/sem_seg.py | 2 +- vis4d/zoo/base/datasets/bdd100k/track.py | 2 +- vis4d/zoo/base/datasets/coco/detection.py | 2 +- vis4d/zoo/base/datasets/coco/sem_seg.py | 2 +- vis4d/zoo/base/datasets/imagenet.py | 2 +- vis4d/zoo/base/models/yolox.py | 2 +- vis4d/zoo/base/optimizer.py | 6 +- vis4d/zoo/base/pl_trainer.py | 2 +- vis4d/zoo/base/runtime.py | 2 +- .../faster_rcnn/faster_rcnn_r50_1x_bdd100k.py | 2 +- .../faster_rcnn/faster_rcnn_r50_3x_bdd100k.py | 2 +- .../mask_rcnn/mask_rcnn_r50_1x_bdd100k.py | 2 +- .../mask_rcnn/mask_rcnn_r50_3x_bdd100k.py | 2 +- .../mask_rcnn/mask_rcnn_r50_5x_bdd100k.py | 2 +- .../qdtrack_frcnn_r50_fpn_1x_bdd100k.py | 2 +- .../semantic_fpn_r101_80k_bdd100k.py | 4 +- .../semantic_fpn_r50_40k_bdd100k.py | 2 +- .../semantic_fpn_r50_80k_bdd100k.py | 2 +- vis4d/zoo/bevformer/bevformer_base.py | 2 +- vis4d/zoo/bevformer/bevformer_tiny.py | 2 +- vis4d/zoo/bevformer/bevformer_vis.py | 2 +- vis4d/zoo/bevformer/data.py | 2 +- vis4d/zoo/cc_3dt/README.md | 23 ++-- .../cc_3dt_bevformer_base_velo_lstm_nusc.py | 2 +- .../cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py | 2 +- .../cc_3dt_frcnn_r101_fpn_pure_det_nusc.py | 2 +- ...c_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py | 2 +- .../cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py | 2 +- vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py | 16 ++- vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py | 2 +- vis4d/zoo/cc_3dt/data.py | 2 +- .../velo_lstm_bevformer_base_100e_nusc.py | 6 +- .../velo_lstm_frcnn_r101_fpn_100e_nusc.py | 6 +- vis4d/zoo/faster_rcnn/faster_rcnn_coco.py | 10 +- vis4d/zoo/fcn_resnet/fcn_resnet_coco.py | 2 +- vis4d/zoo/mask_rcnn/mask_rcnn_coco.py | 2 +- vis4d/zoo/qdtrack/data_yolox.py | 2 +- .../qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py | 2 +- .../qdtrack/qdtrack_yolox_x_50e_bdd100k.py | 2 +- vis4d/zoo/retinanet/retinanet_coco.py | 2 +- .../faster_rcnn/faster_rcnn_r50_12e_shift.py | 2 +- .../faster_rcnn/faster_rcnn_r50_36e_shift.py | 2 +- .../faster_rcnn_r50_6e_shift_all_domains.py | 2 +- .../semantic_fpn_r50_160k_shift.py | 2 +- ...semantic_fpn_r50_160k_shift_all_domains.py | 2 +- .../semantic_fpn_r50_40k_shift.py | 2 +- .../semantic_fpn_r50_40k_shift_all_domains.py | 2 +- vis4d/zoo/vit/vit_small_imagenet.py | 2 +- vis4d/zoo/vit/vit_tiny_imagenet.py | 2 +- vis4d/zoo/yolox/data.py | 2 +- vis4d/zoo/yolox/yolox_s_300e_coco.py | 2 +- vis4d/zoo/yolox/yolox_tiny_300e_coco.py | 2 +- 106 files changed, 741 insertions(+), 417 deletions(-) rename vis4d/{zoo => config}/typing.py (99%) create mode 100644 vis4d/model/adapter/flops.py create mode 100644 vis4d/op/loss/cross_entropy.py diff --git a/docs/source/user_guide/faster_rcnn_example.py b/docs/source/user_guide/faster_rcnn_example.py index ab5ec25e2..b639930d2 100644 --- a/docs/source/user_guide/faster_rcnn_example.py +++ b/docs/source/user_guide/faster_rcnn_example.py @@ -10,6 +10,11 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ( + ExperimentConfig, + ExperimentParameters, + ParameterSweepConfig, +) from vis4d.config.sweep import grid_search from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -31,11 +36,6 @@ get_coco_detection_cfg, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ( - ExperimentConfig, - ExperimentParameters, - ParameterSweepConfig, -) def get_config() -> ExperimentConfig: diff --git a/pyproject.toml b/pyproject.toml index 3e9da90e0..f12accf20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,7 @@ plugins = ["numpy.typing.mypy_plugin"] "terminaltables.*", "timm.*", "vis4d_cuda_ops.*", + "fvcore.*", ] ignore_missing_imports = true diff --git a/tests/config/sweep_test.py b/tests/config/sweep_test.py index 29ed1847e..681488adc 100644 --- a/tests/config/sweep_test.py +++ b/tests/config/sweep_test.py @@ -7,7 +7,7 @@ from vis4d.config.replicator import replicate_config from vis4d.config.sweep import grid_search -from vis4d.zoo.typing import ExperimentConfig +from vis4d.config.typing import ExperimentConfig class TestSweep(unittest.TestCase): diff --git a/tests/engine/optim/optimizer_test.py b/tests/engine/optim/optimizer_test.py index 256965c8f..ba1243c88 100644 --- a/tests/engine/optim/optimizer_test.py +++ b/tests/engine/optim/optimizer_test.py @@ -12,9 +12,9 @@ from tests.util import MockModel from vis4d.config import class_config +from vis4d.config.typing import LrSchedulerConfig, ParamGroupCfg from vis4d.engine.optim import LRSchedulerWrapper, PolyLR, set_up_optimizers from vis4d.zoo.base import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.zoo.typing import LrSchedulerConfig, ParamGroupCfg def get_optimizer( diff --git a/tests/eval/common/depth_test.py b/tests/eval/common/depth_test.py index 525d19250..5d05ff2d1 100644 --- a/tests/eval/common/depth_test.py +++ b/tests/eval/common/depth_test.py @@ -101,6 +101,6 @@ def test_precomputed(self) -> None: metrics[DepthEvaluator.KEY_ABS_REL], 1.208, places=3 ) self.assertAlmostEqual( - metrics[DepthEvaluator.KEY_SQ_REL], 4.007, places=3 + metrics[DepthEvaluator.KEY_SQ_REL], 5.635, places=3 ) assert isinstance(log_str, str) diff --git a/tests/eval/kitti/depth_test.py b/tests/eval/kitti/depth_test.py index e2162a57d..1689e7029 100644 --- a/tests/eval/kitti/depth_test.py +++ b/tests/eval/kitti/depth_test.py @@ -82,6 +82,6 @@ def test_precomputed(self) -> None: metrics[KITTIDepthEvaluator.KEY_ABS_REL], 1.208, places=3 ) self.assertAlmostEqual( - metrics[KITTIDepthEvaluator.KEY_SQ_REL], 4.007, places=3 + metrics[KITTIDepthEvaluator.KEY_SQ_REL], 5.635, places=3 ) assert isinstance(log_str, str) diff --git a/tests/op/base/resnet_test.py b/tests/op/base/resnet_test.py index af6211f3e..83b678c7c 100644 --- a/tests/op/base/resnet_test.py +++ b/tests/op/base/resnet_test.py @@ -50,12 +50,12 @@ def test_resnet(): # test norm freezed model.trainable_layers = 5 - model.norm_freezed = True + model.norm_frozen = True model.train() for m in model.modules(): if isinstance(m, _BatchNorm): assert not m.training - model.norm_freezed = False + model.norm_frozen = False model.train() for m in model.modules(): if isinstance(m, _BatchNorm): diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index 12ae687e3..031f308f1 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit 12ae687e3b468de14501867de85a1cb61df4e520 +Subproject commit 031f308f10742dd083ec681a9ea7b7568744cd01 diff --git a/tests/zoo/util.py b/tests/zoo/util.py index 5aff0e514..bc6a276d3 100644 --- a/tests/zoo/util.py +++ b/tests/zoo/util.py @@ -5,7 +5,7 @@ import importlib from tests.util import content_equal -from vis4d.zoo.typing import ExperimentConfig +from vis4d.config.typing import ExperimentConfig def get_config_for_name(config_name: str) -> ExperimentConfig: diff --git a/vis4d/common/imports.py b/vis4d/common/imports.py index 2cb6920a7..c008358b2 100644 --- a/vis4d/common/imports.py +++ b/vis4d/common/imports.py @@ -25,6 +25,7 @@ def package_available(package_name: str) -> bool: MMSEG_AVAILABLE = package_available("mmseg") DETECTRON2_AVAILABLE = package_available("detectron2") TIMM_AVAILABLE = package_available("timm") +FVCORE_AVAILABLE = package_available("fvcore") # datasets WAYMO_AVAILABLE = package_available("waymo") diff --git a/vis4d/common/logging.py b/vis4d/common/logging.py index 6794453a4..c4c62e82b 100644 --- a/vis4d/common/logging.py +++ b/vis4d/common/logging.py @@ -15,6 +15,7 @@ from vis4d.common.distributed import rank_zero_only from vis4d.common.typing import ArgsType +from vis4d.config.typing import ExperimentConfig def _debug(*args: ArgsType, stacklevel: int = 2, **kwargs: ArgsType) -> None: @@ -125,3 +126,14 @@ def setup_logger( fh.setLevel(logging.DEBUG) fh.setFormatter(plain_formatter) logger.addHandler(fh) + + +@rank_zero_only +def dump_config(config: ExperimentConfig, config_file: str) -> None: + """Dump the configuration to a file. + + Args: + config (ExperimentConfig): The configuration to dump. + config_file (str): The path to the file to dump the configuration to. + """ + config.dump(config_file) diff --git a/vis4d/common/typing.py b/vis4d/common/typing.py index 718581705..5e78f0c1f 100644 --- a/vis4d/common/typing.py +++ b/vis4d/common/typing.py @@ -16,24 +16,28 @@ Tensor, ) +NumpyBool = np.bool_ NumpyFloat = Union[np.float32, np.float64] NumpyInt = Union[np.int32, np.int64] -NumpyUInt = np.uint8 # pylint: disable=invalid-name -NumpyBool = np.bool_ +NumpyUInt = Union[ # pylint: disable=invalid-name + np.uint8, np.uint16, np.uint32 +] -NDArrayF64 = npt.NDArray[np.float64] +NDArrayBool = npt.NDArray[np.bool_] NDArrayF32 = npt.NDArray[np.float32] -NDArrayI64 = npt.NDArray[np.int64] +NDArrayF64 = npt.NDArray[np.float64] +NDArrayFloat = Union[NDArrayF32, NDArrayF64] NDArrayI32 = npt.NDArray[np.int32] +NDArrayI64 = npt.NDArray[np.int64] +NDArrayInt = Union[NDArrayI32, NDArrayI64] NDArrayUI8 = npt.NDArray[np.uint8] NDArrayUI16 = npt.NDArray[np.uint16] NDArrayUI32 = npt.NDArray[np.uint32] -NDArrayBool = npt.NDArray[np.bool_] -NDArrayFloat = Union[NDArrayF32, NDArrayF64] -NDArrayInt = Union[NDArrayI32, NDArrayI64] -NDArrayUInt = NDArrayUI8 - +NDArrayUInt = Union[ # pylint: disable=invalid-name + NDArrayUI8, NDArrayUI16, NDArrayUI32 +] NDArrayNumber = Union[NDArrayBool, NDArrayFloat, NDArrayInt, NDArrayUInt] + MetricLogs = Dict[str, Union[float, int, Tensor]] DictStrAny = Dict[str, Any] # type: ignore DictStrArrNested = Dict[str, Union[Tensor, Dict[str, Tensor]]] diff --git a/vis4d/zoo/typing.py b/vis4d/config/typing.py similarity index 99% rename from vis4d/zoo/typing.py rename to vis4d/config/typing.py index 9c8ff8f3a..f9161dfd0 100644 --- a/vis4d/zoo/typing.py +++ b/vis4d/config/typing.py @@ -7,7 +7,7 @@ from ml_collections import ConfigDict, FieldReference from typing_extensions import NotRequired -from vis4d.config.config_dict import FieldConfigDict +from .config_dict import FieldConfigDict class ParamGroupCfg(TypedDict): diff --git a/vis4d/data/const.py b/vis4d/data/const.py index ca961434f..603a0306e 100644 --- a/vis4d/data/const.py +++ b/vis4d/data/const.py @@ -8,6 +8,11 @@ from dataclasses import dataclass from enum import Enum +# A custom value to distinguish instance ID and category ID; need to be greater +# than the number of categories. For a pixel in the panoptic result map: +# panaptic_id = instance_id * INSTANCE_OFFSET + category_id +INSTANCE_OFFSET = 1000 + class AxisMode(Enum): """Enum for choosing among different coordinate frame conventions. @@ -59,13 +64,14 @@ class CommonKeys: categories (NDArrayF32): Class labels of shape [C, ]. boxes2d (NDArrayF32): 2D bounding boxes of shape [N, 4] in xyxy format. - boxes2d_classes (NDArrayI32): Semantic classes of 2D bounding boxes, shape + boxes2d_classes (NDArrayI64): Semantic classes of 2D bounding boxes, shape [N,]. - boxes2d_track_ids (NDArrayI32): Tracking IDs of 2D bounding boxes, + boxes2d_track_ids (NDArrayI64): Tracking IDs of 2D bounding boxes, shape [N,]. instance_masks (NDArrayUI8): Instance segmentation masks of shape [N, H, W]. seg_masks (NDArrayUI8): Semantic segmentation masks [H, W]. + panoptic_masks (NDArrayI64): Panoptic segmentation masks [H, W]. deph_maps (NDArrayF32): Depth maps of shape [H, W]. intrinsics (NDArrayF32): Intrinsic sensor calibration. Shape [3, 3]. @@ -82,7 +88,7 @@ class CommonKeys: instances3d: TODO complete boxes3d (NDArrayF32): [N, 10], each row consists of center (XYZ), dimensions (WLH), and orientation quaternion (WXYZ). - boxes3d_classes (NDArrayI32): Associated semantic classes of 3D bounding + boxes3d_classes (NDArrayI64): Associated semantic classes of 3D bounding boxes, [N,]. """ @@ -103,6 +109,7 @@ class CommonKeys: boxes2d_track_ids = "boxes2d_track_ids" instance_masks = "instance_masks" seg_masks = "seg_masks" + panoptic_masks = "panoptic_masks" depth_maps = "depth_maps" optical_flows = "optical_flows" diff --git a/vis4d/data/datasets/nuscenes.py b/vis4d/data/datasets/nuscenes.py index 71d7edfa7..eda37a15e 100644 --- a/vis4d/data/datasets/nuscenes.py +++ b/vis4d/data/datasets/nuscenes.py @@ -484,7 +484,7 @@ def _load_can_bus_data( """Load can bus data.""" try: pose_list = can_bus_data.get_messages(scene_name, "pose") - except AssertionError: + except: # pylint: disable=bare-except # server scenes do not have can bus information. return [0.0] * 18 @@ -956,6 +956,8 @@ def __getitem__(self, idx: int) -> DictData: data_dict[cam][K.input_hw] = cam_data["image_hw"] data_dict[cam][K.sample_names] = cam_data["sample_name"] data_dict[cam][K.intrinsics] = cam_data["intrinsics"] + data_dict[cam][K.extrinsics] = cam_data["extrinsics"] + data_dict[cam][K.axis_mode] = AxisMode.OPENCV if K.original_images in self.keys_to_load: data_dict[cam][K.original_images] = image @@ -981,8 +983,6 @@ def __getitem__(self, idx: int) -> DictData: data_dict[cam]["attributes"] = cam_data["annotations"][ "boxes3d_attributes" ] - data_dict[cam][K.extrinsics] = cam_data["extrinsics"] - data_dict[cam][K.axis_mode] = AxisMode.OPENCV if K.boxes2d in self.keys_to_load: boxes2d = cam_data["annotations"]["boxes2d"] diff --git a/vis4d/data/transforms/affine.py b/vis4d/data/transforms/affine.py index 72eeee47d..0f6f5cc5d 100644 --- a/vis4d/data/transforms/affine.py +++ b/vis4d/data/transforms/affine.py @@ -13,7 +13,7 @@ import torch from vis4d.common.imports import OPENCV_AVAILABLE -from vis4d.common.typing import NDArrayF32, NDArrayI32 +from vis4d.common.typing import NDArrayF32, NDArrayI64 from vis4d.data.const import CommonKeys as K from vis4d.op.box.box2d import bbox_clip, bbox_project @@ -227,15 +227,7 @@ def __call__( height_list: list[int], width_list: list[int], ) -> tuple[list[NDArrayF32], list[tuple[int, int]]]: - """Crop a list of image of dimensions [N, H, W, C]. - - Args: - images (list[NDArrayF32]): The list of image. - crop_box (list[NDArrayI32]): The list of box to crop. - - Returns: - list[NDArrayF32]: List of cropped image according to parameters. - """ + """Affine a list of image of dimensions [N, H, W, C].""" input_hw_list = [] for i, (image, warp_matrix, height, width) in enumerate( zip(images, warp_matrix_list, height_list, width_list) @@ -282,12 +274,12 @@ def __init__(self, bbox_clip_border: bool = True) -> None: def __call__( self, boxes: list[NDArrayF32], - classes: list[NDArrayI32], - track_ids: list[NDArrayI32] | None, + classes: list[NDArrayI64], + track_ids: list[NDArrayI64] | None, warp_matrix_list: list[NDArrayF32], height_list: list[int], width_list: list[int], - ) -> tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32] | None]: + ) -> tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64] | None]: """Apply Affine to 2D bounding boxes.""" for i, (box, class_, warp_matrix, height, width) in enumerate( zip( diff --git a/vis4d/data/transforms/crop.py b/vis4d/data/transforms/crop.py index 043165046..a1fa1154e 100644 --- a/vis4d/data/transforms/crop.py +++ b/vis4d/data/transforms/crop.py @@ -10,7 +10,13 @@ import torch from vis4d.common.logging import rank_zero_warn -from vis4d.common.typing import NDArrayBool, NDArrayF32, NDArrayI32, NDArrayUI8 +from vis4d.common.typing import ( + NDArrayBool, + NDArrayF32, + NDArrayI32, + NDArrayI64, + NDArrayUI8, +) from vis4d.data.const import CommonKeys as K from vis4d.op.box.box2d import bbox_intersection @@ -100,8 +106,8 @@ def __init__( shape: CropShape, crop_func: CropFunc = absolute_crop, allow_empty_crops: bool = True, - recompute_boxes2d: bool = False, cat_max_ratio: float = 1.0, + ignore_index: int = 255, ) -> None: """Creates an instance of the class. @@ -111,16 +117,15 @@ def __init__( of the crop. Defaults to absolute_crop. allow_empty_crops (bool, optional): Allow crops which result in empty labels. Defaults to True. - recompute_boxes2d (bool, optional): Recompute the bounding boxes - after cropping instance masks. Defaults to False. cat_max_ratio (float, optional): Maximum ratio of a particular class in segmentation masks after cropping. Defaults to 1.0. + ignore_index (int, optional): The index to ignore. Defaults to 255. """ self.shape = shape self.crop_func = crop_func - self.cat_max_ratio = cat_max_ratio self.allow_empty_crops = allow_empty_crops - self.recompute_boxes2d = recompute_boxes2d + self.cat_max_ratio = cat_max_ratio + self.ignore_index = ignore_index def _get_crop( self, im_h: int, im_w: int, boxes: NDArrayF32 | None = None @@ -128,11 +133,7 @@ def _get_crop( """Get the crop parameters.""" crop_size = self.crop_func(im_h, im_w, self.shape) crop_box = _sample_crop(im_h, im_w, crop_size) - keep_mask = ( - _get_keep_mask(boxes, crop_box) - if boxes is not None - else np.array([]) - ) + keep_mask = _get_keep_mask(boxes, crop_box) return crop_box, keep_mask def __call__( @@ -147,14 +148,15 @@ def __call__( masks = masks_list[0] if masks_list is not None else None crop_box, keep_mask = self._get_crop(im_h, im_w, boxes) - if (boxes is not None and len(boxes) > 0) or self.cat_max_ratio != 1.0: + if (boxes is not None and len(boxes) > 0) or masks is not None: # resample crop if conditions not satisfied found_crop = False for _ in range(10): # try resampling 10 times, otherwise use last crop if (self.allow_empty_crops or keep_mask.sum() != 0) and ( - masks is None - or _check_seg_max_cat(masks, crop_box, self.cat_max_ratio) + _check_seg_max_cat( + masks, crop_box, self.cat_max_ratio, self.ignore_index + ) ): found_crop = True break @@ -199,11 +201,7 @@ def __call__( crop_size = self.crop_func(im_h, im_w, self.shape) crop_box = _get_central_crop(im_h, im_w, crop_size) - keep_mask = ( - _get_keep_mask(boxes, crop_box) - if boxes is not None - else np.array([]) - ) + keep_mask = _get_keep_mask(boxes, crop_box) crop_params = [ CropParam(crop_box=crop_box, keep_mask=keep_mask) ] * len(input_hw_list) @@ -282,11 +280,7 @@ def __call__( boxes = boxes_list[0] if boxes_list is not None else None crop_box = self.get_params(im_h, im_w) - keep_mask = ( - _get_keep_mask(boxes, crop_box) - if boxes is not None - else np.array([]) - ) + keep_mask = _get_keep_mask(boxes, crop_box) crop_params = [ CropParam(crop_box=crop_box, keep_mask=keep_mask) @@ -340,25 +334,25 @@ class CropBoxes2D: def __call__( self, boxes_list: list[NDArrayF32], - classes_list: list[NDArrayI32], - track_ids_list: list[NDArrayI32] | None, + classes_list: list[NDArrayI64], + track_ids_list: list[NDArrayI64] | None, crop_box_list: list[NDArrayI32], keep_mask_list: list[NDArrayBool], - ) -> tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32] | None]: + ) -> tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64] | None]: """Crop 2D bounding boxes. Args: boxes_list (list[NDArrayF32]): The list of bounding boxes to be cropped. - classes_list (list[NDArrayI32]): The list of the corresponding + classes_list (list[NDArrayI64]): The list of the corresponding classes. - track_ids_list (list[NDArrayI32] | None, optional): The list of + track_ids_list (list[NDArrayI64] | None, optional): The list of corresponding tracking IDs. Defaults to None. crop_box_list (list[NDArrayI32]): The list of box to crop. keep_mask_list (list[NDArrayBool]): Which boxes to keep. Returns: - tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32]] | None: + tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64]] | None: List of cropped bounding boxes according to parameters. """ for i, (boxes, classes, crop_box, keep_mask) in enumerate( @@ -395,6 +389,31 @@ def __call__( return masks_list +@Transform( + in_keys=[ + K.instance_masks, + "transforms.crop.crop_box", + "transforms.crop.keep_mask", + ], + out_keys=[K.instance_masks], +) +class CropInstanceMasks: + """Crop instance segmentation masks.""" + + def __call__( + self, + masks_list: list[NDArrayUI8], + crop_box_list: list[NDArrayI32], + keep_mask_list: list[NDArrayBool], + ) -> list[NDArrayUI8]: + """Crop masks.""" + for i, (masks, crop_box) in enumerate(zip(masks_list, crop_box_list)): + x1, y1, x2, y2 = crop_box + masks = masks[:, y1:y2, x1:x2] + masks_list[i] = masks[keep_mask_list[i]] + return masks_list + + @Transform([K.depth_maps, "transforms.crop.crop_box"], K.depth_maps) class CropDepthMaps: """Crop depth maps.""" @@ -470,9 +489,11 @@ def _get_central_crop( return np.array([crop_x1, crop_y1, crop_x2, crop_y2]) -def _get_keep_mask(boxes: NDArrayF32, crop_box: NDArrayI32) -> NDArrayBool: +def _get_keep_mask( + boxes: NDArrayF32 | None, crop_box: NDArrayI32 +) -> NDArrayBool: """Get mask for 2D annotations to keep.""" - if len(boxes) == 0: + if boxes is None or len(boxes) == 0: return np.array([], dtype=bool) # will be better to compute mask intersection (if exists) instead overlap = bbox_intersection( @@ -482,21 +503,27 @@ def _get_keep_mask(boxes: NDArrayF32, crop_box: NDArrayI32) -> NDArrayBool: def _check_seg_max_cat( - masks: NDArrayUI8, crop_box: NDArrayI32, cat_max_ratio: float + masks: NDArrayUI8 | None, + crop_box: NDArrayI32, + cat_max_ratio: float, + ignore_index: int = 255, ) -> bool: """Check if any category occupies more than cat_max_ratio. Args: - masks (NDArrayUI8): Segmentation masks. + masks (NDArrayUI8 | None): Segmentation masks. crop_box (NDArrayI32): The box to crop. cat_max_ratio (float): Maximum category ratio. + ignore_index (int, optional): The index to ignore. Defaults to 255. Returns: bool: True if no category occupies more than cat_max_ratio. """ + if cat_max_ratio >= 1.0 or masks is None: + return True x1, y1, x2, y2 = crop_box crop_masks = masks[y1:y2, x1:x2] cls_ids, cnts = np.unique(crop_masks, return_counts=True) - cnts = cnts[cls_ids != 255] - keep_mask = len(cnts) > 1 and cnts.max() / cnts.sum() < cat_max_ratio - return keep_mask + cnts = cnts[cls_ids != ignore_index] + + return (cnts.max() / cnts.sum()) < cat_max_ratio diff --git a/vis4d/data/transforms/mask.py b/vis4d/data/transforms/mask.py index 18867617b..d2b422b64 100644 --- a/vis4d/data/transforms/mask.py +++ b/vis4d/data/transforms/mask.py @@ -4,7 +4,7 @@ import numpy as np -from vis4d.common.typing import NDArrayI32, NDArrayUI8 +from vis4d.common.typing import NDArrayI64, NDArrayUI8 from vis4d.data.const import CommonKeys as K from .base import Transform @@ -18,7 +18,7 @@ class ConvertInstanceMaskToSegMask: """Merge all instance masks into a single segmentation map.""" def __call__( - self, classes_list: list[NDArrayI32], masks_list: list[NDArrayUI8] + self, classes_list: list[NDArrayI64], masks_list: list[NDArrayUI8] ) -> list[NDArrayUI8]: """Execute conversion. @@ -58,7 +58,7 @@ def __init__(self, mapping: list[int]): """ self.mapping = mapping - def __call__(self, classes_list: list[NDArrayI32]) -> list[NDArrayI32]: + def __call__(self, classes_list: list[NDArrayI64]) -> list[NDArrayI64]: """Execute remapping. Args: diff --git a/vis4d/data/transforms/mixup.py b/vis4d/data/transforms/mixup.py index 9e0963a93..584d45462 100644 --- a/vis4d/data/transforms/mixup.py +++ b/vis4d/data/transforms/mixup.py @@ -8,7 +8,7 @@ import numpy as np import torch -from vis4d.common.typing import NDArrayF32, NDArrayI32 +from vis4d.common.typing import NDArrayF32, NDArrayI64 from vis4d.data.const import CommonKeys as K from vis4d.op.box.box2d import bbox_intersection @@ -286,10 +286,10 @@ def __init__( def __call__( self, boxes_list: list[NDArrayF32], - classes_list: list[NDArrayI32], - track_ids_list: list[NDArrayI32] | None, + classes_list: list[NDArrayI64], + track_ids_list: list[NDArrayI64] | None, mixup_parameters: list[MixupParam], - ) -> tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32] | None]: + ) -> tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64] | None]: """Execute the boxes2d mixup operation.""" batch_size = len(boxes_list) assert ( @@ -298,7 +298,7 @@ def __call__( mixup_boxes_list = [] mixup_classes_list = [] - mixup_track_ids_list: list[NDArrayI32] | None = ( + mixup_track_ids_list: list[NDArrayI64] | None = ( [] if track_ids_list is not None else None ) for i in range(0, batch_size, self.NUM_SAMPLES): @@ -350,7 +350,7 @@ def __call__( ) other_track_ids += self.max_track_ids other_track_ids = other_track_ids[is_overlap > 0] - mixup_track_ids: NDArrayI32 = np.concatenate( + mixup_track_ids: NDArrayI64 = np.concatenate( (ori_track_ids, other_track_ids), 0 ) mixup_track_ids_list += [ diff --git a/vis4d/data/transforms/mosaic.py b/vis4d/data/transforms/mosaic.py index f0fae4de5..60845b868 100644 --- a/vis4d/data/transforms/mosaic.py +++ b/vis4d/data/transforms/mosaic.py @@ -10,7 +10,7 @@ import numpy as np -from vis4d.common.typing import NDArrayF32, NDArrayI32 +from vis4d.common.typing import NDArrayF32, NDArrayI64 from vis4d.data.const import CommonKeys as K from .base import Transform @@ -296,15 +296,15 @@ def __init__( def __call__( self, boxes: list[NDArrayF32], - classes: list[NDArrayI32], - track_ids: list[NDArrayI32] | None, + classes: list[NDArrayI64], + track_ids: list[NDArrayI64] | None, paste_coords: list[list[tuple[int, int, int, int]]], crop_coords: list[list[tuple[int, int, int, int]]], im_scales: list[list[tuple[float, float]]], - ) -> tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32] | None]: + ) -> tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64] | None]: """Apply Mosaic to 2D bounding boxes.""" new_boxes, new_classes = [], [] - new_track_ids: list[NDArrayI32] | None = ( + new_track_ids: list[NDArrayI64] | None = ( [] if track_ids is not None else None ) for i in range(0, len(boxes), self.NUM_SAMPLES): diff --git a/vis4d/data/transforms/photometric.py b/vis4d/data/transforms/photometric.py index a8f12d9c1..63c39f522 100644 --- a/vis4d/data/transforms/photometric.py +++ b/vis4d/data/transforms/photometric.py @@ -6,7 +6,7 @@ import numpy as np import torch -import torchvision.transforms.functional as TF +import torchvision.transforms.v2.functional as TF from torch import Tensor from vis4d.common.imports import OPENCV_AVAILABLE diff --git a/vis4d/data/transforms/post_process.py b/vis4d/data/transforms/post_process.py index 7a36a729b..b6605c201 100644 --- a/vis4d/data/transforms/post_process.py +++ b/vis4d/data/transforms/post_process.py @@ -4,7 +4,7 @@ import torch -from vis4d.common.typing import NDArrayF32, NDArrayI32 +from vis4d.common.typing import NDArrayF32, NDArrayI64 from vis4d.data.const import CommonKeys as K from vis4d.op.box.box2d import bbox_area, bbox_clip @@ -50,19 +50,19 @@ def __init__( def __call__( self, boxes_list: list[NDArrayF32], - classes_list: list[NDArrayI32], - track_ids_list: list[NDArrayI32] | None, + classes_list: list[NDArrayI64], + track_ids_list: list[NDArrayI64] | None, input_hw_list: list[tuple[int, int]], boxes3d_list: list[NDArrayF32] | None, - boxes3d_classes_list: list[NDArrayI32] | None, - boxes3d_track_ids_list: list[NDArrayI32] | None, + boxes3d_classes_list: list[NDArrayI64] | None, + boxes3d_track_ids_list: list[NDArrayI64] | None, ) -> tuple[ list[NDArrayF32], - list[NDArrayI32], - list[NDArrayI32] | None, + list[NDArrayI64], + list[NDArrayI64] | None, list[NDArrayF32] | None, - list[NDArrayI32] | None, - list[NDArrayI32] | None, + list[NDArrayI64] | None, + list[NDArrayI64] | None, ]: """Post process according to boxes2D after transformation. @@ -70,32 +70,32 @@ def __call__( boxes_list (list[NDArrayF32]): The bounding boxes to be post processed. classes_list (list[NDArrayF32]): The classes of the bounding boxes. - track_ids_list (list[NDArrayI32] | None): The track ids of the + track_ids_list (list[NDArrayI64] | None): The track ids of the bounding boxes. input_hw_list (list[tuple[int, int]]): The height and width of the input image. boxes3d_list (list[NDArrayF32] | None): The 3D bounding boxes to be post processed. - boxes3d_classes_list (list[NDArrayI32] | None): The classes of the + boxes3d_classes_list (list[NDArrayI64] | None): The classes of the 3D bounding boxes. - boxes3d_track_ids_list (list[NDArrayI32] | None): The track ids of + boxes3d_track_ids_list (list[NDArrayI64] | None): The track ids of the 3D bounding boxes. Returns: - tuple[list[NDArrayF32], list[NDArrayI32], list[NDArrayI32] | None, - list[NDArrayF32] | None, list[NDArrayI32] | None, - list[NDArrayI32] | None]: The post processed results. + tuple[list[NDArrayF32], list[NDArrayI64], list[NDArrayI64] | None, + list[NDArrayF32] | None, list[NDArrayI64] | None, + list[NDArrayI64] | None]: The post processed results. """ - new_track_ids: list[NDArrayI32] | None = ( + new_track_ids: list[NDArrayI64] | None = ( [] if track_ids_list is not None else None ) new_boxes3d: list[NDArrayF32] | None = ( [] if boxes3d_list is not None else None ) - new_boxes3d_classes: list[NDArrayI32] | None = ( + new_boxes3d_classes: list[NDArrayI64] | None = ( [] if boxes3d_classes_list is not None else None ) - new_boxes3d_track_ids: list[NDArrayI32] | None = ( + new_boxes3d_track_ids: list[NDArrayI64] | None = ( [] if boxes3d_track_ids_list is not None else None ) for i, (boxes, classes) in enumerate(zip(boxes_list, classes_list)): @@ -138,15 +138,15 @@ def __call__( class RescaleTrackIDs: """Rescale track ids.""" - def __call__(self, track_ids_list: list[NDArrayI32]) -> list[NDArrayI32]: + def __call__(self, track_ids_list: list[NDArrayI64]) -> list[NDArrayI64]: """Rescale the track ids. Args: - track_ids_list (list[NDArrayI32]): The track ids to be + track_ids_list (list[NDArrayI64]): The track ids to be rescaled. Returns: - list[NDArrayI32]: The rescaled track ids. + list[NDArrayI64]: The rescaled track ids. """ track_ids_all: dict[int, int] = {} for track_ids in track_ids_list: diff --git a/vis4d/data/transforms/resize.py b/vis4d/data/transforms/resize.py index b5556d1c7..4c4eff1ae 100644 --- a/vis4d/data/transforms/resize.py +++ b/vis4d/data/transforms/resize.py @@ -46,6 +46,7 @@ def __init__( multiscale_mode: str = "range", scale_range: tuple[float, float] = (1.0, 1.0), align_long_edge: bool = False, + resize_short_edge: bool = False, allow_overflow: bool = False, ) -> None: """Creates an instance of the class. @@ -67,6 +68,10 @@ def __init__( long edge of the original image, e.g. original shape=(100, 80), shape to be resized=(100, 200) will yield (125, 100) as new shape. Defaults to False. + resize_short_edge (bool, optional): If keep_ratio=true, this option + scale the image according to the short edge. e.g. original + shape=(80, 100), shape to be resized=(100, 200) will yield + (100, 125) as new shape. Defaults to False. allow_overflow (bool, optional): If set to True, we scale the image to the smallest size such that it is no smaller than shape. Otherwise, we scale the image to the largest size such that it @@ -77,6 +82,7 @@ def __init__( self.multiscale_mode = multiscale_mode self.scale_range = scale_range self.align_long_edge = align_long_edge + self.resize_short_edge = resize_short_edge self.allow_overflow = allow_overflow def __call__( @@ -93,6 +99,7 @@ def __call__( self.multiscale_mode, self.scale_range, self.align_long_edge, + self.resize_short_edge, self.allow_overflow, ) scale_factor = ( @@ -425,6 +432,7 @@ def get_resize_shape( new_shape: tuple[int, int], keep_ratio: bool = True, align_long_edge: bool = False, + resize_short_edge: bool = False, allow_overflow: bool = False, ) -> tuple[int, int]: """Get shape for resize, considering keep_ratio and align_long_edge. @@ -437,6 +445,8 @@ def get_resize_shape( align_long_edge (bool, optional): Whether to align the long edge of the original shape with the long edge of the new shape. Defaults to False. + resize_short_edge (bool, optional): Whether to resize according to the + short edge. Defaults to False. allow_overflow (bool, optional): Whether to allow overflow. Defaults to False. @@ -455,6 +465,10 @@ def get_resize_shape( scale_factor = comp_fn( long_edge / max(h, w), short_edge / min(h, w) ) + elif resize_short_edge: + short_edge = min(original_shape) + new_short_edge = min(new_shape) + scale_factor = new_short_edge / short_edge else: scale_factor = comp_fn(new_w / w, new_h / h) new_h = int(h * scale_factor + 0.5) @@ -469,6 +483,7 @@ def get_target_shape( multiscale_mode: str = "range", scale_range: tuple[float, float] = (1.0, 1.0), align_long_edge: bool = False, + resize_short_edge: bool = False, allow_overflow: bool = False, ) -> tuple[int, int]: """Generate possibly random target shape.""" @@ -506,6 +521,11 @@ def get_target_shape( shape = random.choice(shape) shape = get_resize_shape( - input_shape, shape, keep_ratio, align_long_edge, allow_overflow + input_shape, + shape, + keep_ratio, + align_long_edge, + resize_short_edge, + allow_overflow, ) return shape diff --git a/vis4d/data/transforms/to_tensor.py b/vis4d/data/transforms/to_tensor.py index 2e42c258b..c3ec5e3e3 100644 --- a/vis4d/data/transforms/to_tensor.py +++ b/vis4d/data/transforms/to_tensor.py @@ -13,7 +13,17 @@ def _replace_arrays(data: DictData) -> None: """Replace numpy arrays with tensors.""" for key in data.keys(): if key in [K.images, K.original_images]: - data[key] = torch.from_numpy(data[key]).permute(0, 3, 1, 2) + if not data[key].flags.c_contiguous: + data[key] = np.ascontiguousarray( + data[key].transpose(0, 3, 1, 2) + ) + data[key] = torch.from_numpy(data[key]) + else: + data[key] = ( + torch.from_numpy(data[key]) + .permute(0, 3, 1, 2) + .contiguous() + ) elif isinstance(data[key], np.ndarray): data[key] = torch.from_numpy(data[key]) elif isinstance(data[key], dict): diff --git a/vis4d/engine/callbacks/evaluator.py b/vis4d/engine/callbacks/evaluator.py index 82ef5e0dc..989b005cb 100644 --- a/vis4d/engine/callbacks/evaluator.py +++ b/vis4d/engine/callbacks/evaluator.py @@ -7,7 +7,12 @@ from torch import nn from vis4d.common import ArgsType, MetricLogs -from vis4d.common.distributed import all_gather_object_cpu, broadcast, get_rank +from vis4d.common.distributed import ( + all_gather_object_cpu, + broadcast, + rank_zero_only, + synchronize, +) from vis4d.common.logging import rank_zero_info from vis4d.data.typing import DictData from vis4d.eval.base import Evaluator @@ -84,14 +89,13 @@ def on_test_epoch_end( """Hook to run at the end of a testing epoch.""" self.evaluator.gather(all_gather_object_cpu) - if get_rank() == 0: - log_dict = self.evaluate() - else: # pragma: no cover - log_dict = None + synchronize() + log_dict = self.evaluate() log_dict = broadcast(log_dict) self.evaluator.reset() return log_dict + @rank_zero_only def evaluate(self) -> MetricLogs: """Evaluate the performance after processing all input/output pairs. @@ -117,7 +121,8 @@ def evaluate(self) -> MetricLogs: for k, v in metric_dict.items(): log_k = metric + "/" + k rank_zero_info("%s: %.4f", log_k, v) - log_dict[log_k] = v + log_dict[f"{metric}/{k}"] = v + rank_zero_info("Showing results for metric: %s", metric) rank_zero_info(metric_str) diff --git a/vis4d/engine/experiment.py b/vis4d/engine/experiment.py index eae6dadbc..10cac2200 100644 --- a/vis4d/engine/experiment.py +++ b/vis4d/engine/experiment.py @@ -24,6 +24,7 @@ ) from vis4d.common.logging import ( _info, + dump_config, rank_zero_info, rank_zero_warn, setup_logger, @@ -31,7 +32,7 @@ from vis4d.common.slurm import init_dist_slurm from vis4d.common.util import init_random_seed, set_random_seed, set_tf32 from vis4d.config import instantiate_classes -from vis4d.zoo.typing import ExperimentConfig +from vis4d.config.typing import ExperimentConfig from .optim import set_up_optimizers from .parser import pprints_config @@ -107,6 +108,12 @@ def run_experiment( log_dir = os.path.join(config.output_dir, f"log_{config.timestamp}.txt") setup_logger(logger_vis4d, log_dir) + # Dump config + config_file = os.path.join( + config.output_dir, f"config_{config.timestamp}.yaml" + ) + dump_config(config, config_file) + rank_zero_info("Environment info: %s", get_pretty_env_info()) # PyTorch Setting diff --git a/vis4d/engine/optim/optimizer.py b/vis4d/engine/optim/optimizer.py index 23adf9832..33629ee5b 100644 --- a/vis4d/engine/optim/optimizer.py +++ b/vis4d/engine/optim/optimizer.py @@ -13,7 +13,7 @@ from vis4d.common.logging import rank_zero_info from vis4d.config import instantiate_classes -from vis4d.zoo.typing import OptimizerConfig, ParamGroupCfg +from vis4d.config.typing import OptimizerConfig, ParamGroupCfg from .scheduler import LRSchedulerWrapper diff --git a/vis4d/engine/optim/scheduler.py b/vis4d/engine/optim/scheduler.py index 79effc206..674f4bf2f 100644 --- a/vis4d/engine/optim/scheduler.py +++ b/vis4d/engine/optim/scheduler.py @@ -9,7 +9,7 @@ from vis4d.common.typing import DictStrAny from vis4d.config import copy_and_resolve_references, instantiate_classes -from vis4d.zoo.typing import LrSchedulerConfig +from vis4d.config.typing import LrSchedulerConfig class LRSchedulerDict(TypedDict): @@ -146,8 +146,6 @@ class ConstantLR(LRScheduler): max_steps (int): Maximum number of steps. factor (float): Scale factor. Default: 1.0 / 3.0. last_epoch (int): The index of last epoch. Default: -1. - verbose (bool): If ``True``, prints a message to stdout for each - update. Default: ``False``. """ def __init__( @@ -156,12 +154,11 @@ def __init__( max_steps: int, factor: float = 1.0 / 3.0, last_epoch: int = -1, - verbose: bool = False, ): """Initialize ConstantLR.""" self.max_steps = max_steps self.factor = factor - super().__init__(optimizer, last_epoch, verbose) + super().__init__(optimizer, last_epoch) def get_lr(self) -> list[float]: # type: ignore """Compute current learning rate.""" @@ -197,8 +194,6 @@ class PolyLR(LRScheduler): power (float, optional): Power factor. Default: 1.0. min_lr (float): Minimum learning rate. Default: 0.0. last_epoch (int): The index of last epoch. Default: -1. - verbose (bool): If ``True``, prints a message to stdout for each - update. Default: ``False``. """ def __init__( @@ -208,13 +203,12 @@ def __init__( power: float = 1.0, min_lr: float = 0.0, last_epoch: int = -1, - verbose: bool = False, ): """Initialize PolyLRScheduler.""" self.max_steps = max_steps self.power = power self.min_lr = min_lr - super().__init__(optimizer, last_epoch, verbose) + super().__init__(optimizer, last_epoch) def get_lr(self) -> list[float]: # type: ignore """Compute current learning rate.""" @@ -238,8 +232,6 @@ class QuadraticLRWarmup(LRScheduler): optimizer (Optimizer): Wrapped optimizer. max_steps (int): Maximum number of steps. last_epoch (int): The index of last epoch. Default: -1. - verbose (bool): If ``True``, prints a message to stdout for each - update. Default: ``False``. """ def __init__( @@ -247,11 +239,10 @@ def __init__( optimizer: Optimizer, max_steps: int, last_epoch: int = -1, - verbose: bool = False, ): """Initialize QuadraticLRWarmup.""" self.max_steps = max_steps - super().__init__(optimizer, last_epoch, verbose) + super().__init__(optimizer, last_epoch) def get_lr(self) -> list[float]: # type: ignore """Compute current learning rate.""" diff --git a/vis4d/engine/run.py b/vis4d/engine/run.py index 47512a3f6..406ab3cb0 100644 --- a/vis4d/engine/run.py +++ b/vis4d/engine/run.py @@ -8,7 +8,7 @@ from vis4d.common.logging import rank_zero_info from vis4d.config import instantiate_classes from vis4d.config.replicator import replicate_config -from vis4d.zoo.typing import ExperimentConfig +from vis4d.config.typing import ExperimentConfig from .experiment import run_experiment from .flag import _CKPT, _CONFIG, _GPUS, _RESUME, _SHOW_CONFIG, _SLURM, _SWEEP diff --git a/vis4d/eval/common/depth.py b/vis4d/eval/common/depth.py index 387eedbee..02d7db4b6 100644 --- a/vis4d/eval/common/depth.py +++ b/vis4d/eval/common/depth.py @@ -47,7 +47,7 @@ class DepthEvaluator(Evaluator): def __init__( self, - min_depth: float = 0.01, + min_depth: float = 0.0, max_depth: float = 80.0, scale: float = 1.0, epsilon: float = 1e-3, @@ -68,6 +68,10 @@ def __init__( self.scale = scale self._metrics_list: list[dict[str, float]] = [] + def __repr__(self) -> str: + """Concise representation of the evaluator.""" + return "Common Depth Evaluator" + @property def metrics(self) -> list[str]: """Supported metrics.""" @@ -87,7 +91,7 @@ def _apply_mask( self, prediction: NDArrayFloat, target: NDArrayFloat ) -> tuple[NDArrayFloat, NDArrayFloat]: """Apply mask to prediction and target.""" - mask = (target > self.min_depth) & (target < self.max_depth) + mask = (target > self.min_depth) & (target <= self.max_depth) return prediction[mask], target[mask] def process_batch( # type: ignore # pylint: disable=arguments-differ @@ -147,14 +151,14 @@ def evaluate(self, metric: str) -> tuple[MetricLogs, str]: Register data using .process() first!""" ) metric_data: MetricLogs = {} - short_description = "" + short_description = "\n" if metric == self.METRIC_DEPTH: abs_rel = np.mean( [x[self.KEY_ABS_REL] for x in self._metrics_list] ) metric_data[self.KEY_ABS_REL] = float(abs_rel) - short_description += f"Absolute relative error: {abs_rel:.3f} " + short_description += f"Absolute relative error: {abs_rel:.3f}\n" abs_err = np.mean( [x[self.KEY_ABS_ERR] for x in self._metrics_list] diff --git a/vis4d/eval/metrics/depth.py b/vis4d/eval/metrics/depth.py index 70e8f633d..8d783b6c0 100644 --- a/vis4d/eval/metrics/depth.py +++ b/vis4d/eval/metrics/depth.py @@ -36,7 +36,7 @@ def squared_relative_error(prediction: ArrayLike, target: ArrayLike) -> float: """ prediction, target = dense_inputs_to_numpy(prediction, target) check_shape_match(prediction, target) - return np.mean(np.square(prediction - target) / np.square(target)).item() + return np.mean(np.square(prediction - target) / target).item() def absolute_relative_error(prediction: ArrayLike, target: ArrayLike) -> float: diff --git a/vis4d/model/adapter/flops.py b/vis4d/model/adapter/flops.py new file mode 100644 index 000000000..170178af8 --- /dev/null +++ b/vis4d/model/adapter/flops.py @@ -0,0 +1,59 @@ +"""Adapter for counting flops in a model.""" + +from __future__ import annotations + +from typing import Any + +from torch import nn + +from vis4d.engine.connectors import DataConnector + +# Ops to ignore from counting, including elementwise and reduction ops +IGNORED_OPS = { + "aten::add", + "aten::add_", + "aten::argmax", + "aten::argsort", + "aten::batch_norm", + "aten::constant_pad_nd", + "aten::div", + "aten::div_", + "aten::exp", + "aten::log2", + "aten::max_pool2d", + "aten::meshgrid", + "aten::mul", + "aten::mul_", + "aten::neg", + "aten::nonzero_numpy", + "aten::reciprocal", + "aten::repeat_interleave", + "aten::rsub", + "aten::sigmoid", + "aten::sigmoid_", + "aten::softmax", + "aten::sort", + "aten::sqrt", + "aten::sub", + "torchvision::nms", +} + + +class FlopsModelAdapter(nn.Module): + """Adapter for the model to count flops.""" + + def __init__( + self, model: nn.Module, data_connector: DataConnector + ) -> None: + """Initialize the adapter.""" + super().__init__() + self.model = model + self.data_connector = data_connector + + def forward(self, *args: Any) -> Any: # type: ignore + """Forward pass through the model.""" + data_dict = {} + for i, key in enumerate(self.data_connector.key_mapping): + data_dict[key] = args[0][i] + + return self.model(**data_dict) diff --git a/vis4d/model/seg/semantic_fpn.py b/vis4d/model/seg/semantic_fpn.py index c9b4a8e44..9b8420840 100644 --- a/vis4d/model/seg/semantic_fpn.py +++ b/vis4d/model/seg/semantic_fpn.py @@ -65,7 +65,7 @@ def __init__( "resnet50_v1c", pretrained=True, trainable_layers=5, - norm_freezed=False, + norm_frozen=False, ) self.basemodel = basemodel @@ -92,7 +92,7 @@ def forward_train(self, images: torch.Tensor) -> SemanticFPNOut: features = self.fpn(self.basemodel(images.contiguous())) out = self.seg_head(features) if self.resize: - out = SemanticFPNOut( + return SemanticFPNOut( outputs=F.interpolate( out.outputs, scale_factor=4, diff --git a/vis4d/model/segment3d/pointnetpp.py b/vis4d/model/segment3d/pointnetpp.py index 46a35ba7f..6e7a65d92 100644 --- a/vis4d/model/segment3d/pointnetpp.py +++ b/vis4d/model/segment3d/pointnetpp.py @@ -2,14 +2,12 @@ from __future__ import annotations -from typing import overload - import torch -import torch.nn as nn +from torch import Tensor, nn from vis4d.common.ckpt import load_model_checkpoint from vis4d.common.typing import LossesType, ModelOutput -from vis4d.data.const import CommonKeys +from vis4d.data.const import CommonKeys as K from vis4d.op.base.pointnetpp import ( PointNet2Segmentation, PointNet2SegmentationOut, @@ -41,21 +39,13 @@ def __init__( if weights is not None: load_model_checkpoint(self, weights) - @overload - def forward(self, points3d: torch.Tensor) -> ModelOutput: ... - - @overload - def forward( - self, points3d: torch.Tensor, semantics3d: torch.Tensor | None - ) -> PointNet2SegmentationOut: ... - def forward( - self, points3d: torch.Tensor, semantics3d: torch.Tensor | None = None + self, points3d: Tensor, semantics3d: Tensor | None = None ) -> PointNet2SegmentationOut | ModelOutput: """Forward pass of the model. Extract semantic predictions. Args: - points3d (torch.Tensor): Input point shape [b, N, C]. + points3d (Tensor): Input point shape [b, N, C]. semantics3d (torch.Tenosr): Groundtruth semantic labels of shape [b, N]. Defaults to None @@ -66,30 +56,7 @@ def forward( if semantics3d is not None: return x class_pred = torch.argmax(x.class_logits, dim=1) - return {CommonKeys.semantics3d: class_pred} - - def forward_test(self, points3d: torch.Tensor) -> ModelOutput: - """Forward test. - - Args: - points3d (torch.Tensor): Input point shape [b, N, C]. - - Returns: - ModelOutput: Semantic predictions of the model. - """ - return self.forward(points3d) - - def forward_train( - self, points3d: torch.Tensor, semantics3d: torch.Tensor - ) -> PointNet2SegmentationOut: - """Forward train. - - Args: - points3d (torch.Tensor): Input point shape [b, N, C]. - semantics3d (torch.Tenosr): Groundtruth semantic labels of - shape [b, N]. Defaults to None. - """ - return self.forward(points3d, semantics3d) + return {K.semantics3d: class_pred} class Pointnet2SegmentationLoss(nn.Module): @@ -98,14 +65,14 @@ class Pointnet2SegmentationLoss(nn.Module): def __init__( self, ignore_index: int = 255, - semantic_weights: torch.Tensor | None = None, + semantic_weights: Tensor | None = None, ) -> None: """Creates an instance of the class. Args: ignore_index (int, optional): Class Index that should be ignored. Defaults to 255. - semantic_weights (torch.Tensor, optional): Weights for each class. + semantic_weights (Tensor, optional): Weights for each class. """ super().__init__() self.segmentation_loss = nn.CrossEntropyLoss( @@ -113,13 +80,13 @@ def __init__( ) def forward( - self, outputs: PointNet2SegmentationOut, semantics3d: torch.Tensor + self, outputs: PointNet2SegmentationOut, semantics3d: Tensor ) -> LossesType: """Calculates the loss. Args: outputs (PointNet2SegmentationOut): Model outputs. - semantics3d (torch.Tensor): Groundtruth semantic labels. + semantics3d (Tensor): Groundtruth semantic labels. """ return dict( segmentation_loss=self.segmentation_loss( diff --git a/vis4d/op/base/resnet.py b/vis4d/op/base/resnet.py index 84e51aa2b..0e00bb026 100644 --- a/vis4d/op/base/resnet.py +++ b/vis4d/op/base/resnet.py @@ -5,22 +5,20 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Sequence -import torch.utils.checkpoint as cp import torchvision.models.resnet as _resnet from torch import Tensor, nn from torch.nn.modules.batchnorm import _BatchNorm +from torch.utils.checkpoint import checkpoint from vis4d.common.ckpt import load_model_checkpoint from vis4d.common.typing import ArgsType -from vis4d.op.layer.util import build_conv_layer +from vis4d.op.layer.util import build_conv_layer, build_norm_layer from vis4d.op.layer.weight_init import constant_init, kaiming_init from .base import BaseModel -NormLayerType = Callable[..., nn.Module] # type: ignore - class BasicBlock(nn.Module): """BasicBlock.""" @@ -35,17 +33,14 @@ def __init__( dilation: int = 1, downsample: nn.Module | None = None, style: str = "pytorch", - with_cp: bool = False, + use_checkpoint: bool = False, with_dcn: bool = False, - norm_layer: NormLayerType | None = None, + norm: str = "BatchNorm2d", ) -> None: """Creates an instance of the class.""" super().__init__() assert style in {"pytorch", "caffe"} # No effect for BasicBlock assert not with_dcn, "DCN is not supported for BasicBlock." - if norm_layer is None: - norm_layer = nn.BatchNorm2d - self.conv1 = build_conv_layer( inplanes, planes, @@ -55,15 +50,15 @@ def __init__( padding=dilation, bias=False, ) - self.bn1 = norm_layer(planes) + self.bn1 = build_norm_layer(norm, planes) self.conv2 = build_conv_layer(planes, planes, 3, padding=1, bias=False) - self.bn2 = norm_layer(planes) + self.bn2 = build_norm_layer(norm, planes) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride self.dilation = dilation - self.with_cp = with_cp + self.use_checkpoint = use_checkpoint def forward(self, x: Tensor) -> Tensor: """Forward function.""" @@ -85,8 +80,8 @@ def _inner_forward(x: Tensor) -> Tensor: return out - if self.with_cp and x.requires_grad: - out = cp.checkpoint(_inner_forward, x) + if self.use_checkpoint and x.requires_grad: + out = checkpoint(_inner_forward, x) else: out = _inner_forward(x) @@ -108,9 +103,9 @@ def __init__( dilation: int = 1, downsample: nn.Module | None = None, style: str = "pytorch", - with_cp: bool = False, + use_checkpoint: bool = False, with_dcn: bool = False, - norm_layer: NormLayerType | None = None, + norm: str = "BatchNorm2d", ) -> None: """Bottleneck block for ResNet. @@ -118,14 +113,11 @@ def __init__( it is "caffe", the stride-two layer is the first 1x1 conv layer. """ super().__init__() - if norm_layer is None: - norm_layer = nn.BatchNorm2d - self.inplanes = inplanes self.planes = planes self.stride = stride self.dilation = dilation - self.with_cp = with_cp + self.use_checkpoint = use_checkpoint assert style in {"pytorch", "caffe"} if style == "pytorch": @@ -142,7 +134,7 @@ def __init__( stride=self.conv1_stride, bias=False, ) - self.bn1 = norm_layer(planes) + self.bn1 = build_norm_layer(norm, planes) self.conv2 = build_conv_layer( planes, @@ -154,7 +146,7 @@ def __init__( bias=False, use_dcn=with_dcn, ) - self.bn2 = norm_layer(planes) + self.bn2 = build_norm_layer(norm, planes) self.conv3 = build_conv_layer( planes, @@ -162,7 +154,7 @@ def __init__( kernel_size=1, bias=False, ) - self.bn3 = norm_layer(planes * self.expansion) + self.bn3 = build_norm_layer(norm, planes * self.expansion) self.relu = nn.ReLU(inplace=True) self.downsample = downsample @@ -190,8 +182,8 @@ def _inner_forward(x: Tensor) -> Tensor: return out - if self.with_cp and x.requires_grad: - out = cp.checkpoint(_inner_forward, x) + if self.use_checkpoint and x.requires_grad: + out = checkpoint(_inner_forward, x) else: out = _inner_forward(x) @@ -224,11 +216,11 @@ def __init__( deep_stem: bool = False, avg_down: bool = False, trainable_layers: int = 5, - norm_layer: NormLayerType | None = None, - norm_freezed: bool = True, + norm: str = "BatchNorm2d", + norm_frozen: bool = True, stages_with_dcn: Sequence[bool] = (False, False, False, False), replace_stride_with_dilation: Sequence[bool] = (False, False, False), - with_cp: bool = False, + use_checkpoint: bool = False, zero_init_residual: bool = True, pretrained: bool = False, weights: None | str = None, @@ -258,18 +250,18 @@ def __init__( trainable_layers (int, optional): Number layers for training or fine-tuning. 5 means all the layers can be fine-tuned. Defaults to 5. - norm_layer (Callable[..., nn.Module] | None): Normalization layer. - Default: None, which means using `nn.BatchNorm2d`. - norm_freezed (bool): Whether to set norm layers to eval mode, - namely, freeze running stats (mean and var). Note: Effect on + norm (str): Normalization layer str. Default: BatchNorm2d, which + means using `nn.BatchNorm2d`. + norm_frozen (bool): Whether to set norm layers to eval mode. It + freezes running stats (mean and var). Note: Effect on Batch Norm and its variants only. stages_with_dcn (Sequence[bool]): Indices of stages with deformable convolutions. Default: (False, False, False, False). replace_stride_with_dilation (Sequence[bool]): Whether to replace stride with dilation. Default: (False, False, False). - with_cp (bool): Use checkpoint or not. Using checkpoint will save - some memory while slowing down the training speed. Default: - False. + use_checkpoint (bool): Use checkpoint or not. Using checkpoint will + save some memory while slowing down the training speed. + Default: False. zero_init_residual (bool): Whether to use zero init for last norm layer in resblocks to let them behave as identity. Default: True. @@ -278,9 +270,7 @@ def __init__( weights (str, optional): model pretrained path. Default: None """ super().__init__() - if norm_layer is None: - norm_layer = nn.BatchNorm2d - self._norm_layer = norm_layer + self._norm = norm self.zero_init_residual = zero_init_residual if resnet_name not in self.arch_settings: @@ -289,8 +279,8 @@ def __init__( self.deep_stem = deep_stem self.trainable_layers = trainable_layers - self.with_cp = with_cp - self.norm_freezed = norm_freezed + self.use_checkpoint = use_checkpoint + self.norm_frozen = norm_frozen depth, self.block, stage_blocks = self.arch_settings[resnet_name] assert isinstance(depth, int) @@ -324,7 +314,7 @@ def __init__( dilation=dilation, style=style, avg_down=avg_down, - with_cp=with_cp, + use_checkpoint=use_checkpoint, with_dcn=stages_with_dcn[i], ) self.inplanes = planes * self.block.expansion # type: ignore @@ -332,8 +322,6 @@ def __init__( self.add_module(layer_name, res_layer) self.res_layers.append(layer_name) - self._freeze_stages() - if pretrained: if weights is None: # default loading the imagenet-1k v1 pre-trained model weights @@ -345,6 +333,8 @@ def __init__( else: self._init_weights() + self._freeze_stages() + def _init_weights(self) -> None: """Initialize the weights of module.""" for m in self.modules(): @@ -376,7 +366,7 @@ def _make_stem_layer(self, in_channels: int, stem_channels: int) -> None: padding=1, bias=False, ), - self._norm_layer(stem_channels // 2), + build_norm_layer(self._norm, stem_channels // 2), nn.ReLU(inplace=True), build_conv_layer( stem_channels // 2, @@ -386,7 +376,7 @@ def _make_stem_layer(self, in_channels: int, stem_channels: int) -> None: padding=1, bias=False, ), - self._norm_layer(stem_channels // 2), + build_norm_layer(self._norm, stem_channels // 2), nn.ReLU(inplace=True), build_conv_layer( stem_channels // 2, @@ -396,7 +386,7 @@ def _make_stem_layer(self, in_channels: int, stem_channels: int) -> None: padding=1, bias=False, ), - self._norm_layer(stem_channels), + build_norm_layer(self._norm, stem_channels), nn.ReLU(inplace=True), ) else: @@ -408,7 +398,7 @@ def _make_stem_layer(self, in_channels: int, stem_channels: int) -> None: padding=3, bias=False, ) - self.bn1 = self._norm_layer(stem_channels) + self.bn1 = build_norm_layer(self._norm, stem_channels) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) @@ -422,7 +412,7 @@ def _make_res_layer( dilation: int, style: str, avg_down: bool, - with_cp: bool, + use_checkpoint: bool, with_dcn: bool, ) -> nn.Sequential: """Pack all blocks in a stage into a ``ResLayer``.""" @@ -450,7 +440,7 @@ def _make_res_layer( stride=conv_stride, bias=False, ), - self._norm_layer(planes * block.expansion), + build_norm_layer(self._norm, planes * block.expansion), ] ) downsample = nn.Sequential(*downsample_list) @@ -464,9 +454,9 @@ def _make_res_layer( dilation=dilation, downsample=downsample, style=style, - with_cp=with_cp, + use_checkpoint=use_checkpoint, with_dcn=with_dcn, - norm_layer=self._norm_layer, + norm=self._norm, ) ) inplanes = planes * block.expansion @@ -478,9 +468,9 @@ def _make_res_layer( stride=1, dilation=dilation, style=style, - with_cp=with_cp, + use_checkpoint=use_checkpoint, with_dcn=with_dcn, - norm_layer=self._norm_layer, + norm=self._norm, ) ) return nn.Sequential(*layers) @@ -509,7 +499,7 @@ def train(self, mode: bool = True) -> ResNet: super().train(mode) self._freeze_stages() - if mode and self.norm_freezed: + if mode and self.norm_frozen: for m in self.modules(): # trick: eval have effect on BatchNorm only if isinstance(m, _BatchNorm): @@ -523,7 +513,6 @@ def out_channels(self) -> list[int]: Returns: list[int]: number of channels """ - # use static value to be compatible with torch.jit if self.name in {"resnet18", "resnet34"}: # channels = [3, 3] + [64 * 2**i for i in range(4)] channels = [3, 3, 64, 128, 256, 512] diff --git a/vis4d/op/box/box2d.py b/vis4d/op/box/box2d.py index c1aeb344f..3359c21ed 100644 --- a/vis4d/op/box/box2d.py +++ b/vis4d/op/box/box2d.py @@ -4,8 +4,9 @@ import torch from torch import Tensor -from torchvision.ops import batched_nms +from torchvision.ops import batched_nms, nms +from vis4d.common.logging import rank_zero_warn from vis4d.op.geometry.transform import transform_points @@ -378,8 +379,10 @@ def multiclass_nms( score_thr: float, iou_thr: float, max_num: int = -1, + class_agnostic: bool = False, + split_thr: int = 100000, ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """A per-class version of Non-maximum suppression. + """Non-maximum suppression with multiple classes. Args: multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) @@ -390,6 +393,11 @@ def multiclass_nms( iou_thr (float): NMS IoU threshold max_num (int, optional): if there are more than max_num bboxes after NMS, only top max_num will be kept. Defaults to -1. + class_agnostic (bool, optional): whether apply class_agnostic NMS. + Defaults to False. + split_thr (int, optional): If the number of bboxes is less than + split_thr, use class agnostic NMS with class_agnostic=True. + Defaults to 100000. Returns: tuple: (Tensor, Tensor, Tensor, Tensor): detections (k, 5), scores @@ -417,12 +425,12 @@ def multiclass_nms( labels = labels.reshape(-1) if not torch.onnx.is_in_onnx_export(): - # NonZero not supported in TensorRT + # NonZero not supported in TensorRT # remove low scoring boxes valid_mask = scores > score_thr if not torch.onnx.is_in_onnx_export(): - # NonZero not supported in TensorRT + # NonZero not supported in TensorRT inds = valid_mask.nonzero(as_tuple=False).squeeze(1) bboxes, scores, labels = bboxes[inds], scores[inds], labels[inds] else: @@ -440,7 +448,15 @@ def multiclass_nms( ) return bboxes, scores, labels, inds - keep = batched_nms(bboxes, scores, labels, iou_thr) + if class_agnostic and bboxes.shape[0] < split_thr: + keep = nms(bboxes, scores, iou_thr) + else: + if class_agnostic: + rank_zero_warn( + f"Number of bboxes is larger than {split_thr}, " + "using per-class NMS instead" + ) + keep = batched_nms(bboxes, scores, labels, iou_thr) if max_num > 0: keep = keep[:max_num] diff --git a/vis4d/op/detect/rcnn.py b/vis4d/op/detect/rcnn.py index d1d0b087b..ddac6ae9e 100644 --- a/vis4d/op/detect/rcnn.py +++ b/vis4d/op/detect/rcnn.py @@ -121,8 +121,6 @@ def _add_conv_fc_branch( is_shared: bool = False, ) -> tuple[nn.ModuleList, nn.ModuleList, int]: """Add shared or separable branch.""" - last_layer_dim = in_channels - # add branch specific conv layers convs, last_layer_dim = add_conv_branch( num_branch_convs, in_channels, @@ -196,6 +194,7 @@ def __init__( score_threshold: float = 0.05, iou_threshold: float = 0.5, max_per_img: int = 100, + class_agnostic_nms: bool = False, ) -> None: """Creates an instance of the class. @@ -209,6 +208,8 @@ def __init__( post-processing step. Defaults to 0.5. max_per_img (int, optional): Maximum number of detections per image. Defaults to 100. + class_agnostic_nms (bool, optional): Whether to use class agnostic + NMS. Defaults to False. """ super().__init__() if box_decoder is None: @@ -218,6 +219,7 @@ def __init__( self.score_threshold = score_threshold self.max_per_img = max_per_img self.iou_threshold = iou_threshold + self.class_agnostic_nms = class_agnostic_nms def forward( self, @@ -259,6 +261,7 @@ def forward( self.score_threshold, self.iou_threshold, self.max_per_img, + self.class_agnostic_nms, ) all_det_boxes.append(det_bbox) all_det_scores.append(det_scores) diff --git a/vis4d/op/detect3d/bevformer/decoder.py b/vis4d/op/detect3d/bevformer/decoder.py index 178521109..c4d14ed64 100644 --- a/vis4d/op/detect3d/bevformer/decoder.py +++ b/vis4d/op/detect3d/bevformer/decoder.py @@ -152,18 +152,13 @@ def __init__( self.embed_dims = embed_dims self.ffns = nn.ModuleList() - - layers: list[nn.Module] = [ - nn.Sequential( - nn.Linear(self.embed_dims, feedforward_channels), - nn.ReLU(inplace=True), - nn.Dropout(drop_out), + self.ffns.append( + FFN( + embed_dims=self.embed_dims, + feedforward_channels=feedforward_channels, + dropout=drop_out, ) - ] - layers.append(nn.Linear(feedforward_channels, self.embed_dims)) - layers.append(nn.Dropout(drop_out)) - - self.ffns.append(FFN(layers=nn.Sequential(*layers))) + ) self.norms = nn.ModuleList() for _ in range(3): diff --git a/vis4d/op/detect3d/bevformer/encoder.py b/vis4d/op/detect3d/bevformer/encoder.py index c0732130f..ddda533ae 100644 --- a/vis4d/op/detect3d/bevformer/encoder.py +++ b/vis4d/op/detect3d/bevformer/encoder.py @@ -350,14 +350,9 @@ def __init__( cross_attn: SpatialCrossAttention | None = None, feedforward_channels: int = 512, drop_out: float = 0.1, - batch_first: bool = True, - pre_norm: bool = False, ) -> None: """Init.""" super().__init__() - self.batch_first = batch_first - self.pre_norm = pre_norm - self.attentions = nn.ModuleList() self_attn = self_attn or TemporalSelfAttention( @@ -371,18 +366,13 @@ def __init__( self.embed_dims = embed_dims self.ffns = nn.ModuleList() - - layers: list[nn.Module] = [ - nn.Sequential( - nn.Linear(self.embed_dims, feedforward_channels), - nn.ReLU(inplace=True), - nn.Dropout(drop_out), + self.ffns.append( + FFN( + embed_dims=embed_dims, + feedforward_channels=feedforward_channels, + dropout=drop_out, ) - ] - layers.append(nn.Linear(feedforward_channels, self.embed_dims)) - layers.append(nn.Dropout(drop_out)) - - self.ffns.append(FFN(layers=nn.Sequential(*layers))) + ) self.norms = nn.ModuleList() for _ in range(3): diff --git a/vis4d/op/detect3d/qd_3dt.py b/vis4d/op/detect3d/qd_3dt.py index 99a7e704e..6d11b167a 100644 --- a/vis4d/op/detect3d/qd_3dt.py +++ b/vis4d/op/detect3d/qd_3dt.py @@ -304,8 +304,6 @@ def _add_conv_fc_branch( is_shared: bool = False, ) -> tuple[nn.ModuleList, nn.ModuleList, int]: """Init modules of head.""" - last_layer_dim = in_channels - # add branch specific conv layers convs, last_layer_dim = add_conv_branch( num_branch_convs, in_channels, diff --git a/vis4d/op/layer/attention.py b/vis4d/op/layer/attention.py index ab60eed92..4a539ee3e 100644 --- a/vis4d/op/layer/attention.py +++ b/vis4d/op/layer/attention.py @@ -119,6 +119,7 @@ def __init__( """ super().__init__() self.batch_first = batch_first + self.embed_dims = embed_dims self.attn = nn.MultiheadAttention( embed_dims, num_heads, dropout=attn_drop, **kwargs @@ -206,7 +207,7 @@ def forward( # ``torch.nn.MultiheadAttention`` is (num_query, batch, # embed_dims), We should adjust the shape of dataflow from # batch_first (batch, num_query, embed_dims) to num_query_first - # (num_query ,batch, embed_dims), and recover ``attn_output`` + # (num_query, batch, embed_dims), and recover ``attn_output`` # from num_query_first to batch_first. if self.batch_first: query = query.transpose(0, 1) diff --git a/vis4d/op/layer/ms_deform_attn.py b/vis4d/op/layer/ms_deform_attn.py index 2d16f70c1..1de35d629 100644 --- a/vis4d/op/layer/ms_deform_attn.py +++ b/vis4d/op/layer/ms_deform_attn.py @@ -221,6 +221,7 @@ def __init__( is_power_of_2(d_model // n_heads) self.d_model = d_model + self.embed_dims = d_model self.n_levels = n_levels self.n_heads = n_heads self.n_points = n_points diff --git a/vis4d/op/layer/transformer.py b/vis4d/op/layer/transformer.py index 614cd8254..08c7950dd 100644 --- a/vis4d/op/layer/transformer.py +++ b/vis4d/op/layer/transformer.py @@ -14,6 +14,7 @@ from .attention import Attention from .drop import DropPath from .mlp import TransformerBlockMLP +from .util import build_activation_layer def inverse_sigmoid(x: Tensor, eps: float = 1e-5) -> Tensor: @@ -181,28 +182,52 @@ class FFN(nn.Module): def __init__( self, - layers: nn.Module, embed_dims: int = 256, - dropout_layer: nn.Module = nn.Identity(), + feedforward_channels: int = 1024, + num_fcs: int = 2, + dropout: float = 0.0, + activation: str = "ReLU", + inplace: bool = True, + dropout_layer: nn.Module | None = None, add_identity: bool = True, layer_scale_init_value: float = 0.0, ) -> None: """Init FFN. Args: - layers (nn.Module): The layers of the FFN. - embed_dims (int): The feature dimension. Same as - `MultiheadAttention`. Defaults: 256. - dropout_layer (nn.Module): The dropout_layer used when adding the - shortcut. + embed_dims (int): The feature dimension. Defaults: 256. + feedforward_channels (int): The hidden dimension of FFNs. + Defaults: 1024. + num_fcs (int): The number of fully-connected layers in FFNs. + Defaults: 2. + dropout (float): The dropout rate of FFNs. + activation (str): The activation function of FFNs. + inplace (bool): Whether to set inplace for activation. + dropout_layer (nn.Module | None, optional): The dropout_layer used + when adding the shortcut. Defaults to None. If None, Identity + is used. add_identity (bool, optional): Whether to add the identity - connection. Default: `True`. + connection. Default: True. layer_scale_init_value (float): Initial value of scale factor in LayerScale. Default: 0.0 """ super().__init__() - self.layers = layers - self.dropout_layer = dropout_layer + layers: list[nn.Module] = [] + in_channels = embed_dims + for _ in range(num_fcs - 1): + layers.append( + nn.Sequential( + nn.Linear(in_channels, feedforward_channels), + build_activation_layer(activation, inplace), + nn.Dropout(dropout), + ) + ) + in_channels = feedforward_channels + layers.append(nn.Linear(feedforward_channels, embed_dims)) + layers.append(nn.Dropout(dropout)) + self.layers = nn.Sequential(*layers) + + self.dropout_layer = dropout_layer or nn.Identity() self.add_identity = add_identity self.layer_scale_init_value = layer_scale_init_value diff --git a/vis4d/op/layer/util.py b/vis4d/op/layer/util.py index c49dc1771..9c6c7488e 100644 --- a/vis4d/op/layer/util.py +++ b/vis4d/op/layer/util.py @@ -4,6 +4,7 @@ from torch import nn +from .conv2d import Conv2d from .deform_conv import DeformConv @@ -16,6 +17,8 @@ def build_conv_layer( dilation: int = 1, groups: int = 1, bias: bool = False, + norm: nn.Module | None = None, + activation: nn.Module | None = None, use_dcn: bool = False, ) -> nn.Module: """Build a convolution layer.""" @@ -29,9 +32,11 @@ def build_conv_layer( dilation=dilation, groups=groups, bias=bias, + norm=norm, + activation=activation, ) - return nn.Conv2d( + return Conv2d( in_planes, out_planes, kernel_size=kernel_size, @@ -40,4 +45,45 @@ def build_conv_layer( dilation=dilation, groups=groups, bias=bias, + norm=norm, + activation=activation, ) + + +def build_activation_layer( + activation: str, inplace: bool = False +) -> nn.Module: + """Build activation layer. + + Args: + activation (str): Activation layer type. + inplace (bool, optional): If to set inplace. Defaults to False. It will + be ignored if the activation layer is not inplace. + """ + activation_layer = getattr(nn, activation) + + if activation_layer in {nn.Tanh, nn.PReLU, nn.Sigmoid, nn.GELU}: + return activation_layer() + + return activation_layer(inplace=inplace) + + +def build_norm_layer( + norm: str, out_channels: int, num_groups: int | None = None +) -> nn.Module: + """Build normalization layer. + + Args: + norm (str): Normalization layer type. + out_channels (int): Number of output channels. + num_groups (int | None, optional): Number of groups for GroupNorm. + Defaults to None. + """ + norm_layer = getattr(nn, norm) + if norm_layer == nn.GroupNorm: + assert ( + num_groups is not None + ), "num_groups must be specified when using Group Norm" + return norm_layer(num_groups, out_channels) + + return norm_layer(out_channels) diff --git a/vis4d/op/loss/cross_entropy.py b/vis4d/op/loss/cross_entropy.py new file mode 100644 index 000000000..e51ead860 --- /dev/null +++ b/vis4d/op/loss/cross_entropy.py @@ -0,0 +1,89 @@ +"""Cross entropy loss.""" + +from __future__ import annotations + +import torch.nn.functional as F +from torch import Tensor + +from .base import Loss +from .reducer import LossReducer, mean_loss + + +class CrossEntropyLoss(Loss): + """Cross entropy loss class.""" + + def __init__( + self, + reducer: LossReducer = mean_loss, + class_weights: list[float] | None = None, + ) -> None: + """Creates an instance of the class. + + Args: + reducer (LossReducer): Reducer for the loss function. Defaults to + mean_loss. + class_weights (list[float], optional): Class weights for the loss + function. Defaults to None. + """ + super().__init__(reducer) + self.class_weights = class_weights + + def forward( + self, + output: Tensor, + target: Tensor, + reducer: LossReducer | None = None, + ignore_index: int = 255, + ) -> Tensor: + """Forward pass. + + Args: + output (list[Tensor]): Model output. + target (Tensor): Assigned segmentation target mask. + reducer (LossReducer, optional): Reducer for the loss function. + Defaults to None. + ignore_index (int): Ignore class id. Default to 255. + + Returns: + Tensor: Computed loss. + """ + if self.class_weights is not None: + class_weights = output.new_tensor( + self.class_weights, device=output.device + ) + else: + class_weights = None + reducer = reducer or self.reducer + + return reducer( + cross_entropy( + output, target, class_weights, ignore_index=ignore_index + ) + ) + + +def cross_entropy( + output: Tensor, + target: Tensor, + class_weights: Tensor | None = None, + ignore_index: int = 255, +) -> Tensor: + """Cross entropy loss function. + + Args: + output (Tensor): Model output. + target (Tensor): Assigned segmentation target mask. + class_weights (Tensor | None, optional): Class weights for the loss + function. Defaults to None. + ignore_index (int): Ignore class id. Default to 255. + + Returns: + Tensor: Computed loss. + """ + return F.cross_entropy( + output, + target.long(), + weight=class_weights, + ignore_index=ignore_index, + reduction="none", + ) diff --git a/vis4d/op/loss/multi_level_seg_loss.py b/vis4d/op/loss/multi_level_seg_loss.py index 7518935be..2c1cddfc9 100644 --- a/vis4d/op/loss/multi_level_seg_loss.py +++ b/vis4d/op/loss/multi_level_seg_loss.py @@ -2,15 +2,14 @@ from __future__ import annotations -from collections.abc import Callable - import torch +from torch import Tensor from vis4d.common.typing import LossesType from .base import Loss +from .cross_entropy import cross_entropy from .reducer import LossReducer, mean_loss -from .seg_cross_entropy_loss import seg_cross_entropy class MultiLevelSegLoss(Loss): @@ -25,9 +24,6 @@ def __init__( self, reducer: LossReducer = mean_loss, feature_idx: tuple[int, ...] = (0,), - loss_fn: Callable[ - [torch.Tensor, torch.Tensor], torch.Tensor - ] = seg_cross_entropy, weights: list[float] | None = None, ) -> None: """Creates an instance of the class. @@ -37,35 +33,40 @@ def __init__( mean_loss. feature_idx (tuple[int]): Indices for the level of features to compute losses. Defaults to (0,). - loss_fn (Callable, optional): Loss function that computes between - predictions and targets. Defaults to seg_cross_entropy. weights (list[float], optional): The weights of each feature level. If None passes, it will set to 1 for all levels. Defaults to None. """ super().__init__(reducer) self.feature_idx = feature_idx - self.loss_fn = loss_fn if weights is None: self.weights = [1.0] * len(self.feature_idx) else: self.weights = weights def forward( - self, outputs: list[torch.Tensor], target: torch.Tensor + self, outputs: list[Tensor], target: Tensor, ignore_index: int = 255 ) -> LossesType: """Forward pass. Args: - outputs (list[torch.Tensor]): Multi-level outputs. - target (torch.Tensor): Assigned segmentation target mask. + outputs (list[Tensor]): Multi-level outputs. + target (Tensor): Assigned segmentation target mask. + ignore_index (int): Ignore class id. Default to 255. Returns: LossesType: Computed losses for each level. """ losses: LossesType = {} + tgt_h, tgt_w = target.shape[-2:] for i, idx in enumerate(self.feature_idx): - loss = self.reducer(self.loss_fn(outputs[idx], target)) + loss = self.reducer( + cross_entropy( + outputs[idx][:, :, :tgt_h, :tgt_w], + target, + ignore_index=ignore_index, + ) + ) losses[f"loss_seg_level{idx}"] = torch.mul(self.weights[i], loss) return losses diff --git a/vis4d/op/loss/seg_cross_entropy_loss.py b/vis4d/op/loss/seg_cross_entropy_loss.py index 26bfb8dde..069affb89 100644 --- a/vis4d/op/loss/seg_cross_entropy_loss.py +++ b/vis4d/op/loss/seg_cross_entropy_loss.py @@ -2,12 +2,12 @@ from __future__ import annotations -import torch -import torch.nn.functional as F +from torch import Tensor from vis4d.common.typing import LossesType from .base import Loss +from .cross_entropy import cross_entropy from .reducer import LossReducer, mean_loss @@ -28,38 +28,23 @@ def __init__(self, reducer: LossReducer = mean_loss) -> None: super().__init__(reducer) def forward( - self, output: torch.Tensor, target: torch.Tensor + self, output: Tensor, target: Tensor, ignore_index: int = 255 ) -> LossesType: """Forward pass. Args: - output (list[torch.Tensor]): Model output. - target (torch.Tensor): Assigned segmentation target mask. + output (list[Tensor]): Model output. + target (Tensor): Assigned segmentation target mask. + ignore_index (int): Ignore class id. Default to 255. Returns: LossesType: Computed loss. """ losses: LossesType = {} - losses["loss_seg"] = self.reducer(seg_cross_entropy(output, target)) + tgt_h, tgt_w = target.shape[-2:] + losses["loss_seg"] = self.reducer( + cross_entropy( + output[:, :, :tgt_h, :tgt_w], target, ignore_index=ignore_index + ) + ) return losses - - -def seg_cross_entropy( - output: torch.Tensor, target: torch.Tensor -) -> torch.Tensor: - """Segmentation cross entropy loss function. - - Args: - output (torch.Tensor): Model output. - target (torch.Tensor): Assigned segmentation target mask. - - Returns: - torch.Tensor: Computed loss. - """ - tgt_h, tgt_w = target.shape[-2:] - return F.cross_entropy( - output[:, :, :tgt_h, :tgt_w], - target.long(), - ignore_index=255, - reduction="none", - ) diff --git a/vis4d/op/mask/util.py b/vis4d/op/mask/util.py index 4c7018e7c..5723cacb4 100644 --- a/vis4d/op/mask/util.py +++ b/vis4d/op/mask/util.py @@ -255,3 +255,26 @@ def postprocess_segms( ).squeeze(1) ) return torch.stack(post_segms).argmax(dim=1) + + +def masks2boxes(masks: Tensor) -> Tensor: + """Obtain the tight bounding boxes of binary masks. + + Args: + masks (Tensor): Binary mask of shape (N, H, W). + + Returns: + Tensor: Boxes with shape (N, 4) of positive region in binary mask. + """ + num_masks = masks.shape[0] + bboxes = masks.new_zeros((num_masks, 4), dtype=torch.float32) + x_any = torch.any(masks, dim=1) + y_any = torch.any(masks, dim=2) + for i in range(num_masks): + x = torch.where(x_any[i, :])[0] + y = torch.where(y_any[i, :])[0] + if len(x) > 0 and len(y) > 0: + bboxes[i, :] = bboxes.new_tensor( + [x[0], y[0], x[-1] + 1, y[-1] + 1] + ) + return bboxes diff --git a/vis4d/op/seg/semantic_fpn.py b/vis4d/op/seg/semantic_fpn.py index 1cfd9720b..b646fda3d 100644 --- a/vis4d/op/seg/semantic_fpn.py +++ b/vis4d/op/seg/semantic_fpn.py @@ -4,9 +4,8 @@ from typing import NamedTuple -import torch import torch.nn.functional as F -from torch import nn +from torch import Tensor, nn from vis4d.op.layer.conv2d import Conv2d @@ -14,7 +13,7 @@ class SemanticFPNOut(NamedTuple): """Output of the SemanticFPN prediction.""" - outputs: torch.Tensor # logits for final prediction, (N, C, H, W) + outputs: Tensor # logits for final prediction, (N, C, H, W) class SemanticFPNHead(nn.Module): @@ -91,11 +90,11 @@ def init_weights(self) -> None: if hasattr(self.conv_seg, "bias") and self.conv_seg.bias is not None: nn.init.constant_(self.conv_seg.bias, 0) - def forward(self, features: list[torch.Tensor]) -> SemanticFPNOut: + def forward(self, features: list[Tensor]) -> SemanticFPNOut: """Transforms feature maps and returns segmentation prediction. Args: - features (list[torch.Tensor]): List of multi-level image features. + features (list[Tensor]): List of multi-level image features. Returns: SemanticFPNOut: Segmentation outputs. @@ -117,3 +116,7 @@ def forward(self, features: list[torch.Tensor]) -> SemanticFPNOut: output = self.dropout(output) seg_preds = self.conv_seg(output) return SemanticFPNOut(outputs=seg_preds) + + def __call__(self, feats: list[Tensor]) -> SemanticFPNOut: + """Type definition for function call.""" + return super()._call_impl(feats) diff --git a/vis4d/pl/data_module.py b/vis4d/pl/data_module.py index 58dd8504d..de307a6e1 100644 --- a/vis4d/pl/data_module.py +++ b/vis4d/pl/data_module.py @@ -6,8 +6,8 @@ from torch.utils.data import DataLoader from vis4d.config import instantiate_classes +from vis4d.config.typing import DataConfig from vis4d.data.typing import DictData -from vis4d.zoo.typing import DataConfig class DataModule(pl.LightningDataModule): diff --git a/vis4d/pl/run.py b/vis4d/pl/run.py index e4c76bef2..c24924dab 100644 --- a/vis4d/pl/run.py +++ b/vis4d/pl/run.py @@ -12,17 +12,17 @@ from torch.utils.collect_env import get_pretty_env_info from vis4d.common import ArgsType -from vis4d.common.logging import rank_zero_info, setup_logger +from vis4d.common.logging import dump_config, rank_zero_info, setup_logger from vis4d.common.util import set_tf32 from vis4d.config import instantiate_classes -from vis4d.engine.callbacks.checkpoint import CheckpointCallback +from vis4d.config.typing import ExperimentConfig +from vis4d.engine.callbacks import CheckpointCallback from vis4d.engine.flag import _CKPT, _CONFIG, _GPUS, _RESUME, _SHOW_CONFIG from vis4d.engine.parser import pprints_config from vis4d.pl.callbacks import CallbackWrapper, LRSchedulerCallback from vis4d.pl.data_module import DataModule from vis4d.pl.trainer import PLTrainer from vis4d.pl.training_module import TrainingModule -from vis4d.zoo.typing import ExperimentConfig def main(argv: ArgsType) -> None: @@ -44,6 +44,12 @@ def main(argv: ArgsType) -> None: setup_logger(logger_vis4d, log_file) setup_logger(logger_pl, log_file) + # Dump config + config_file = osp.join( + config.output_dir, f"config_{config.timestamp}.yaml" + ) + dump_config(config, config_file) + rank_zero_info("Environment info: %s", get_pretty_env_info()) # PyTorch Setting diff --git a/vis4d/pl/training_module.py b/vis4d/pl/training_module.py index 79ba8a2c1..218921639 100644 --- a/vis4d/pl/training_module.py +++ b/vis4d/pl/training_module.py @@ -11,15 +11,20 @@ from vis4d.common.ckpt import load_model_checkpoint from vis4d.common.distributed import broadcast +from vis4d.common.imports import FVCORE_AVAILABLE from vis4d.common.logging import rank_zero_info from vis4d.common.typing import DictStrAny from vis4d.common.util import init_random_seed from vis4d.config import instantiate_classes +from vis4d.config.typing import OptimizerConfig from vis4d.data.typing import DictData from vis4d.engine.connectors import DataConnector from vis4d.engine.loss_module import LossModule from vis4d.engine.optim import LRSchedulerWrapper, set_up_optimizers -from vis4d.zoo.typing import OptimizerConfig +from vis4d.model.adapter.flops import IGNORED_OPS, FlopsModelAdapter + +if FVCORE_AVAILABLE: + from fvcore.nn import FlopCountAnalysis class TrainingModule(pl.LightningModule): @@ -39,6 +44,7 @@ def __init__( hyper_parameters: DictStrAny | None = None, seed: int = -1, ckpt_path: None | str = None, + compute_flops: bool = False, ) -> None: """Initialize the TrainingModule. @@ -54,6 +60,9 @@ def __init__( seed (int, optional): The integer value seed for global random state. Defaults to -1. If -1, a random seed will be generated. ckpt_path (str, optional): The path to the checkpoint to load. + Defaults to None. + compute_flops (bool, optional): If to compute the FLOPs of the + model. Defaults to False. """ super().__init__() self.model_cfg = model_cfg @@ -64,6 +73,7 @@ def __init__( self.hyper_parameters = hyper_parameters self.seed = seed self.ckpt_path = ckpt_path + self.compute_flops = compute_flops # Create model placeholder self.model: nn.Module @@ -134,6 +144,25 @@ def test_step( # pylint: disable=arguments-differ,line-too-long,unused-argument ) -> DictData: """Perform a single test step.""" assert self.test_data_connector is not None + + if self.compute_flops: + flatten_inputs = [ + self.test_data_connector(batch)[key] + for key in self.test_data_connector(batch) + ] + + flops_model = FlopsModelAdapter( + self.model, self.test_data_connector + ) + + flop_analyzer = FlopCountAnalysis(flops_model, flatten_inputs) + + flop_analyzer.set_op_handle(**{k: None for k in IGNORED_OPS}) + + flops = flop_analyzer.total() / 1e9 + + rank_zero_info(f"Flops: {flops:.2f} Gflops") + out = self.model(**self.test_data_connector(batch)) return out diff --git a/vis4d/vis/image/util.py b/vis4d/vis/image/util.py index 846414aa8..c2a20afbd 100644 --- a/vis4d/vis/image/util.py +++ b/vis4d/vis/image/util.py @@ -289,7 +289,9 @@ def preprocess_masks( Raises: ValueError: If the masks have an invalid shape. """ - masks_np = array_to_numpy(masks, n_dims=None, dtype=np.uint8) + masks_np: NDArrayUI8 = array_to_numpy( # type: ignore + masks, n_dims=None, dtype=np.uint8 + ) if len(masks_np.shape) == 2: masks_np, class_ids = _to_binary_mask(masks_np) diff --git a/vis4d/zoo/base/datasets/bdd100k/detect.py b/vis4d/zoo/base/datasets/bdd100k/detect.py index 78eff06a1..a37e2b3ce 100644 --- a/vis4d/zoo/base/datasets/bdd100k/detect.py +++ b/vis4d/zoo/base/datasets/bdd100k/detect.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets import BDD100K @@ -31,7 +32,6 @@ get_inference_dataloaders_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig CONN_BDD100K_DET_EVAL = { "frame_ids": data_key("frame_ids"), diff --git a/vis4d/zoo/base/datasets/bdd100k/sem_seg.py b/vis4d/zoo/base/datasets/bdd100k/sem_seg.py index eb7e0616d..999332069 100644 --- a/vis4d/zoo/base/datasets/bdd100k/sem_seg.py +++ b/vis4d/zoo/base/datasets/bdd100k/sem_seg.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.bdd100k import BDD100K @@ -32,7 +33,6 @@ get_inference_dataloaders_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig CONN_BDD100K_SEG_EVAL = { "data_names": data_key("sample_names"), diff --git a/vis4d/zoo/base/datasets/bdd100k/track.py b/vis4d/zoo/base/datasets/bdd100k/track.py index 8a4190c97..35fccf811 100644 --- a/vis4d/zoo/base/datasets/bdd100k/track.py +++ b/vis4d/zoo/base/datasets/bdd100k/track.py @@ -5,6 +5,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets import BDD100K, bdd100k_track_map @@ -25,7 +26,6 @@ get_inference_dataloaders_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig CONN_BDD100K_TRACK_EVAL = { "frame_ids": data_key("frame_ids"), diff --git a/vis4d/zoo/base/datasets/coco/detection.py b/vis4d/zoo/base/datasets/coco/detection.py index fe99eaf74..7dad6c896 100644 --- a/vis4d/zoo/base/datasets/coco/detection.py +++ b/vis4d/zoo/base/datasets/coco/detection.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.coco import COCO @@ -31,7 +32,6 @@ get_inference_dataloaders_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig CONN_COCO_BBOX_EVAL = { "coco_image_id": data_key(K.sample_names), diff --git a/vis4d/zoo/base/datasets/coco/sem_seg.py b/vis4d/zoo/base/datasets/coco/sem_seg.py index 8bc2c9f58..9ce3467f0 100644 --- a/vis4d/zoo/base/datasets/coco/sem_seg.py +++ b/vis4d/zoo/base/datasets/coco/sem_seg.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.coco import COCO @@ -26,7 +27,6 @@ get_inference_dataloaders_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig def get_train_dataloader( diff --git a/vis4d/zoo/base/datasets/imagenet.py b/vis4d/zoo/base/datasets/imagenet.py index a1b4a7e5e..9668bba25 100644 --- a/vis4d/zoo/base/datasets/imagenet.py +++ b/vis4d/zoo/base/datasets/imagenet.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.datasets.imagenet import ImageNet from vis4d.data.transforms.autoaugment import RandAug @@ -28,7 +29,6 @@ from vis4d.data.transforms.to_tensor import ToTensor from vis4d.engine.connectors import data_key, pred_key from vis4d.zoo.base import get_train_dataloader_cfg -from vis4d.zoo.typing import DataConfig CONN_IMAGENET_CLS_EVAL = { "prediction": pred_key("probs"), diff --git a/vis4d/zoo/base/models/yolox.py b/vis4d/zoo/base/models/yolox.py index fa8946a97..91a18b513 100644 --- a/vis4d/zoo/base/models/yolox.py +++ b/vis4d/zoo/base/models/yolox.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import CosineAnnealingLR from vis4d.config import class_config +from vis4d.config.typing import OptimizerConfig from vis4d.data.const import CommonKeys as K from vis4d.engine.callbacks import ( EMACallback, @@ -23,7 +24,6 @@ from vis4d.op.detect.yolox import YOLOXHead, YOLOXHeadLoss from vis4d.op.fpp import YOLOXPAFPN from vis4d.zoo.base import get_lr_scheduler_cfg, get_optimizer_cfg -from vis4d.zoo.typing import OptimizerConfig # Data connectors CONN_YOLOX_LOSS_2D = { diff --git a/vis4d/zoo/base/optimizer.py b/vis4d/zoo/base/optimizer.py index 3a23ed58c..9d5bebcb5 100644 --- a/vis4d/zoo/base/optimizer.py +++ b/vis4d/zoo/base/optimizer.py @@ -4,7 +4,11 @@ from ml_collections import ConfigDict -from vis4d.zoo.typing import LrSchedulerConfig, OptimizerConfig, ParamGroupCfg +from vis4d.config.typing import ( + LrSchedulerConfig, + OptimizerConfig, + ParamGroupCfg, +) def get_lr_scheduler_cfg( diff --git a/vis4d/zoo/base/pl_trainer.py b/vis4d/zoo/base/pl_trainer.py index f05f979a6..e54e625cb 100644 --- a/vis4d/zoo/base/pl_trainer.py +++ b/vis4d/zoo/base/pl_trainer.py @@ -5,7 +5,7 @@ from lightning import Trainer from vis4d.config import FieldConfigDict -from vis4d.zoo.typing import ExperimentConfig +from vis4d.config.typing import ExperimentConfig def get_default_pl_trainer_cfg(config: ExperimentConfig) -> ExperimentConfig: diff --git a/vis4d/zoo/base/runtime.py b/vis4d/zoo/base/runtime.py index 1fba3dd45..65e842f97 100644 --- a/vis4d/zoo/base/runtime.py +++ b/vis4d/zoo/base/runtime.py @@ -8,8 +8,8 @@ from ml_collections import ConfigDict, FieldReference from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig from vis4d.engine.callbacks import CheckpointCallback, LoggingCallback -from vis4d.zoo.typing import ExperimentConfig def get_default_cfg( diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py index 310417cc4..d92196693 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -30,7 +31,6 @@ get_bdd100k_detection_config, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py index 0ef427dfa..7abc3a7c1 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -30,7 +31,6 @@ get_bdd100k_detection_config, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py index 472239c71..8fc5385d9 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -31,7 +32,6 @@ get_bdd100k_detection_config, ) from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py index 073f2bccb..9a709d213 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -31,7 +32,6 @@ get_bdd100k_detection_config, ) from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py index 2845cf6c2..9c6dec315 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -31,7 +32,6 @@ get_bdd100k_detection_config, ) from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py b/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py index 7f4b79ae3..9b047d2d3 100644 --- a/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/qdtrack/qdtrack_frcnn_r50_fpn_1x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.bdd100k import bdd100k_track_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -29,7 +30,6 @@ CONN_BBOX_2D_TRAIN, get_qdtrack_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py index 277105196..db45092d3 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -38,7 +39,6 @@ CONN_BDD100K_SEG_EVAL, get_bdd100k_sem_seg_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: @@ -90,7 +90,7 @@ def get_config() -> ExperimentConfig: resnet_name="resnet101_v1c", pretrained=True, trainable_layers=5, - norm_freezed=False, + norm_frozen=False, ) config.model = class_config( SemanticFPN, num_classes=params.num_classes, basemodel=basemodel diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py index b1a3ede4f..deb8f78fe 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -37,7 +38,6 @@ CONN_BDD100K_SEG_EVAL, get_bdd100k_sem_seg_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py index cd0657845..06b95cd37 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -37,7 +38,6 @@ CONN_BDD100K_SEG_EVAL, get_bdd100k_sem_seg_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/bevformer_base.py b/vis4d/zoo/bevformer/bevformer_base.py index 28c22c599..c31202022 100644 --- a/vis4d/zoo/bevformer/bevformer_base.py +++ b/vis4d/zoo/bevformer/bevformer_base.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import CallbackConnector, MultiSensorDataConnector @@ -26,7 +27,6 @@ get_nusc_cfg, nuscenes_class_map, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/bevformer_tiny.py b/vis4d/zoo/bevformer/bevformer_tiny.py index 289330e2e..942455183 100644 --- a/vis4d/zoo/bevformer/bevformer_tiny.py +++ b/vis4d/zoo/bevformer/bevformer_tiny.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback from vis4d.engine.connectors import CallbackConnector, MultiSensorDataConnector @@ -37,7 +38,6 @@ get_nusc_cfg, nuscenes_class_map, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/bevformer_vis.py b/vis4d/zoo/bevformer/bevformer_vis.py index c75b30a11..fee098fea 100644 --- a/vis4d/zoo/bevformer/bevformer_vis.py +++ b/vis4d/zoo/bevformer/bevformer_vis.py @@ -3,6 +3,7 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig from vis4d.engine.callbacks import VisualizerCallback from vis4d.engine.connectors import MultiSensorCallbackConnector from vis4d.vis.image.bbox3d_visualizer import MultiCameraBBox3DVisualizer @@ -15,7 +16,6 @@ NUSC_CAMERAS, nuscenes_class_map, ) -from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/bevformer/data.py b/vis4d/zoo/bevformer/data.py index 937ef46ba..d72f74bd1 100644 --- a/vis4d/zoo/bevformer/data.py +++ b/vis4d/zoo/bevformer/data.py @@ -5,6 +5,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.loader import multi_sensor_collate @@ -23,7 +24,6 @@ get_nusc_mini_val_cfg, get_nusc_val_cfg, ) -from vis4d.zoo.typing import DataConfig nuscenes_class_map = { "car": 0, diff --git a/vis4d/zoo/cc_3dt/README.md b/vis4d/zoo/cc_3dt/README.md index 8229a5d8e..29d13638e 100644 --- a/vis4d/zoo/cc_3dt/README.md +++ b/vis4d/zoo/cc_3dt/README.md @@ -11,17 +11,24 @@ This is the official implementation of our paper **"CC-3DT: Panoramic 3D Object To track the 3D locations and trajectories of the other traffic participants at any given time, modern autonomous vehicles are equipped with multiple cameras that cover the vehicle’s full surroundings. Yet, camera-based 3D object tracking methods prioritize optimizing the single-camera setup and resort to post-hoc fusion in a multi-camera setup. In this paper, we propose a method for panoramic 3D object tracking, called CC-3DT, that associates and models object trajectories both temporally and across views, and improves the overall tracking consistency. In particular, our method fuses 3D detections from multiple cameras before association, reducing identity switches significantly and improving motion modeling. Our experiments on large-scale driving datasets show that fusion before association leads to a large margin of improvement over post-hoc fusion. We set a new state-of-the-art with 12.6% improvement in average multi-object tracking accuracy (AMOTA) among all camera-based methods on the competitive NuScenes 3D tracking benchmark, outperforming previously published methods by 6.5% in AMOTA with the same 3D detector. ## Model Zoo +We fixed some inference bugs in the original implementation. The updated results are shown below. So it will be slightly different from the results in the paper. ### CC-3DT with QD-3DT detector -| Backbone | Motion Model | Lr schd | NDS-val | AMOTA-val | Config | Weights | Preds | Visuals | -| :------: | :----------: | :-----: | :-----: | :-------: | :----: | :-----: | :---: | :-----: | -| R-50-FPN | KF3D | 1x | 0.3460 | 0.218 | [config](./cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py) | [tracking model](https://dl.cv.ethz.ch/vis4d/cc_3dt/cc_3dt_frcnn_r50_fpn_12e_nusc_d98509.pt) | [preds]() | [visuals]() | -| R-101-FPN | KF3D | 2x | 0.3790 | 0.302 | [config](./cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py) | [tracking model](https://dl.cv.ethz.ch/vis4d/cc_3dt/cc_3dt_frcnn_r101_fpn_24e_nusc_f24f84.pt) | [preds]() | [visuals]() | -| R-101-FPN | VeloLSTM | 2x | 0.3914 | 0.311 | [config](./cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py) | [motion model](https://dl.cv.ethz.ch/vis4d/cc_3dt/velo_lstm_cc_3dt_frcnn_r101_fpn_100e_nusc_9526a7.pt) | [preds]() | [visuals]() | +| Backbone | Motion Model | Lr schd | NDS-val | AMOTA-val | Config | Weights | +| :------: | :----------: | :-----: | :-----: | :-------: | :----: | :-----: | +| R-50-FPN | KF3D | 1x | 0.3458 | 0.219 | [config](./cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py) | [tracking model](https://dl.cv.ethz.ch/vis4d/cc_3dt/cc_3dt_frcnn_r50_fpn_12e_nusc_d98509.pt) | +| R-101-FPN | KF3D | 2x | 0.3812 | 0.306 | [config](./cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py) | [tracking model](https://dl.cv.ethz.ch/vis4d/cc_3dt/cc_3dt_frcnn_r101_fpn_24e_nusc_f24f84.pt) | +| R-101-FPN | VeloLSTM | 2x | 0.3941 | 0.312 | [config](./cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py) | [motion model](https://dl.cv.ethz.ch/vis4d/cc_3dt/velo_lstm_cc_3dt_frcnn_r101_fpn_100e_nusc_9526a7.pt) | ### CC-3DT with given 3D detections -| Detector | Motion Model | NDS-val | AMOTA-val | Config | Weights | Preds | Visuals | -| :------: | :----------: | :-----: | :-----: | :-------: | :----: | :-----: | :---: | -| BEVFormer | VeloLSTM | 0.4848 | 0.429 | [config](./cc_3dt_bevformer_base_velo_lstm_nusc.py) | [motion model](https://dl.cv.ethz.ch/vis4d/cc_3dt/velo_lstm_bevformer_100e_nusc_1c358e.pt) | [preds]() | [visuals]() | +| Detector | Motion Model | NDS-val | AMOTA-val | Config | Weights | +| :------: | :----------: | :-----: | :-------: | :----: | :-----: | +| BEVFormer (Base) | VeloLSTM | 0.4844 | 0.434 | [config](./cc_3dt_bevformer_base_velo_lstm_nusc.py) | [motion model](https://dl.cv.ethz.ch/vis4d/cc_3dt/velo_lstm_bevformer_100e_nusc_1c358e.pt) | + +We provide the converted [BEVFormer detection results](https://dl.cv.ethz.ch/vis4d/cc_3dt/bevformer_base) for the CC-3DT model as following: +- `bevformer_base.json`: Training set. +- `bevformer_base_val.json`: Validation set. +- `bevformer_base_mini_val.json`: Mini validation set. +- `bevformer_base_test.json`: Test set. ## Getting Started ### Train the Tracking Model diff --git a/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py index 6c93f4576..ddf9740fc 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py @@ -3,6 +3,7 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import DataConfig, ExperimentConfig from vis4d.data.const import CommonKeys as K from vis4d.data.datasets.nuscenes import NuScenes from vis4d.data.datasets.nuscenes_detection import NuScenesDetection @@ -17,7 +18,6 @@ get_config as get_velo_lstm_cfg, ) from vis4d.zoo.cc_3dt.data import CONN_NUSC_BBOX_3D_TEST, get_test_dataloader -from vis4d.zoo.typing import DataConfig, ExperimentConfig CONN_NUSC_BBOX_3D_TEST = { "images_list": data_key(K.images, sensors=NuScenes.CAMERAS), diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py index 61efb82e5..6f685615e 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -34,7 +35,6 @@ get_nusc_cfg, ) from vis4d.zoo.cc_3dt.model import CONN_BBOX_3D_TRAIN, get_cc_3dt_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py index c2e047034..ae44762cb 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py @@ -3,6 +3,7 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import NuScenes, nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -15,7 +16,6 @@ ) from vis4d.zoo.cc_3dt.data import CONN_NUSC_DET3D_EVAL, get_nusc_cfg from vis4d.zoo.cc_3dt.model import get_cc_3dt_cfg -from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py index 5027cea66..343ede71b 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py @@ -3,6 +3,7 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.model.motion.velo_lstm import VeloLSTM from vis4d.op.base import ResNet @@ -10,7 +11,6 @@ get_config as get_kf3d_cfg, ) from vis4d.zoo.cc_3dt.model import get_cc_3dt_cfg -from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py index d9e289039..1354b44a0 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.nuscenes import nuscenes_class_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback @@ -34,7 +35,6 @@ get_nusc_cfg, ) from vis4d.zoo.cc_3dt.model import CONN_BBOX_3D_TRAIN, get_cc_3dt_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py index fa7b8966a..27a2bd047 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py @@ -3,14 +3,12 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import DataConfig, ExperimentConfig from vis4d.data.const import CommonKeys as K -from vis4d.data.datasets.nuscenes import NuScenes +from vis4d.data.datasets.nuscenes_detection import NuScenesDetection from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback -from vis4d.engine.connectors import ( - CallbackConnector, - MultiSensorCallbackConnector, -) +from vis4d.engine.connectors import CallbackConnector from vis4d.eval.nuscenes import ( NuScenesDet3DEvaluator, NuScenesTrack3DEvaluator, @@ -24,7 +22,6 @@ CONN_NUSC_TRACK3D_EVAL, get_test_dataloader, ) -from vis4d.zoo.typing import DataConfig, ExperimentConfig def get_config() -> ExperimentConfig: @@ -50,12 +47,13 @@ def get_config() -> ExperimentConfig: data.train_dataloader = None test_dataset = class_config( - NuScenes, + NuScenesDetection, data_root="data/nuscenes", version="v1.0-test", split="test", keys_to_load=[K.images, K.original_images], data_backend=class_config(HDF5Backend), + pure_detection=config.pure_detection, cache_as_binary=True, cached_file_path="data/nuscenes/test.pkl", ) @@ -86,9 +84,7 @@ def get_config() -> ExperimentConfig: save_predictions=True, save_prefix=config.output_dir, test_connector=class_config( - MultiSensorCallbackConnector, - key_mapping=CONN_NUSC_DET3D_EVAL, - sensors=NuScenes.CAMERAS, + CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL ), ) ) diff --git a/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py b/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py index 5a0b8428a..af086411e 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_nusc_vis.py @@ -3,6 +3,7 @@ from __future__ import annotations from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig from vis4d.data.datasets.nuscenes import NuScenes, nuscenes_class_map from vis4d.engine.callbacks import VisualizerCallback from vis4d.engine.connectors import MultiSensorCallbackConnector @@ -16,7 +17,6 @@ CONN_NUSC_BBOX_3D_VIS, CONN_NUSC_BEV_BBOX_3D_VIS, ) -from vis4d.zoo.typing import ExperimentConfig def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/cc_3dt/data.py b/vis4d/zoo/cc_3dt/data.py index 7cae8b4ad..749ac2bb5 100644 --- a/vis4d/zoo/cc_3dt/data.py +++ b/vis4d/zoo/cc_3dt/data.py @@ -5,6 +5,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe from vis4d.data.datasets.nuscenes import NuScenes @@ -39,7 +40,6 @@ get_nusc_train_cfg, get_nusc_val_cfg, ) -from vis4d.zoo.typing import DataConfig CONN_NUSC_DET3D_EVAL = { "tokens": data_key("token"), diff --git a/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py b/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py index cc9be21f2..55d6eac90 100644 --- a/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py +++ b/vis4d/zoo/cc_3dt/velo_lstm_bevformer_base_100e_nusc.py @@ -7,6 +7,11 @@ from torch.optim.lr_scheduler import MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ( + DataConfig, + ExperimentConfig, + ExperimentParameters, +) from vis4d.data.datasets.nuscenes_trajectory import NuScenesTrajectory from vis4d.engine.connectors import ( DataConnector, @@ -25,7 +30,6 @@ get_optimizer_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig, ExperimentConfig, ExperimentParameters TRAJ_TRAIN = {"pred_traj": "pred_traj"} TRAJ_LOSS = { diff --git a/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py b/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py index 43b1d0f98..01a8dbe08 100644 --- a/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py +++ b/vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py @@ -7,6 +7,11 @@ from torch.optim.lr_scheduler import MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ( + DataConfig, + ExperimentConfig, + ExperimentParameters, +) from vis4d.data.datasets.nuscenes_trajectory import NuScenesTrajectory from vis4d.engine.connectors import ( DataConnector, @@ -25,7 +30,6 @@ get_optimizer_cfg, get_train_dataloader_cfg, ) -from vis4d.zoo.typing import DataConfig, ExperimentConfig, ExperimentParameters TRAJ_TRAIN = {"pred_traj": "pred_traj"} TRAJ_LOSS = { diff --git a/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py b/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py index e3a166ff9..005bedb81 100644 --- a/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py +++ b/vis4d/zoo/faster_rcnn/faster_rcnn_coco.py @@ -9,6 +9,11 @@ from vis4d.config import class_config from vis4d.config.sweep import grid_search +from vis4d.config.typing import ( + ExperimentConfig, + ExperimentParameters, + ParameterSweepConfig, +) from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -32,11 +37,6 @@ get_coco_detection_cfg, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ( - ExperimentConfig, - ExperimentParameters, - ParameterSweepConfig, -) def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py b/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py index 2c41c8ce5..1f177c3fc 100644 --- a/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py +++ b/vis4d/zoo/fcn_resnet/fcn_resnet_coco.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.connectors import DataConnector, LossConnector from vis4d.engine.loss_module import LossModule @@ -26,7 +27,6 @@ CONN_MULTI_SEG_LOSS, ) from vis4d.zoo.base.datasets.coco import get_coco_sem_seg_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py index 7379cc866..58816730a 100644 --- a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py +++ b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -35,7 +36,6 @@ get_coco_detection_cfg, ) from vis4d.zoo.base.models.mask_rcnn import get_mask_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/qdtrack/data_yolox.py b/vis4d/zoo/qdtrack/data_yolox.py index 9ce8b9bb7..376654492 100644 --- a/vis4d/zoo/qdtrack/data_yolox.py +++ b/vis4d/zoo/qdtrack/data_yolox.py @@ -5,6 +5,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe, MultiSampleDataPipe from vis4d.data.datasets.bdd100k import BDD100K, bdd100k_track_map @@ -47,7 +48,6 @@ from vis4d.data.transforms.to_tensor import ToTensor from vis4d.zoo.base import get_inference_dataloaders_cfg from vis4d.zoo.base.callable import get_callable_cfg -from vis4d.zoo.typing import DataConfig def get_train_dataloader( diff --git a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py index 05322b2e5..a49ca5462 100644 --- a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.bdd100k import bdd100k_track_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import ( @@ -33,7 +34,6 @@ get_qdtrack_cfg, ) from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py index e19f82562..60454d369 100644 --- a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py @@ -6,6 +6,7 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.bdd100k import bdd100k_track_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -29,7 +30,6 @@ get_yolox_optimizers_cfg, ) from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/retinanet/retinanet_coco.py b/vis4d/zoo/retinanet/retinanet_coco.py index b4974e157..5ab9d42d5 100644 --- a/vis4d/zoo/retinanet/retinanet_coco.py +++ b/vis4d/zoo/retinanet/retinanet_coco.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -40,7 +41,6 @@ CONN_COCO_BBOX_EVAL, get_coco_detection_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py index e533d7bb5..1e452102d 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -30,7 +31,6 @@ get_shift_det_config, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py index 54bc2314b..946e1fb61 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -30,7 +31,6 @@ get_shift_det_config, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py index 6b549994b..7d84cae75 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR, MultiStepLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import CallbackConnector, DataConnector @@ -30,7 +31,6 @@ get_shift_det_config, ) from vis4d.zoo.base.models.faster_rcnn import get_faster_rcnn_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py index aa330afb4..983d8fef3 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,7 +36,6 @@ CONN_SEG_VIS, ) from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py index b76fdbeae..6fa1b785f 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,7 +36,6 @@ CONN_SEG_VIS, ) from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py index 29cf9d960..bc86ea101 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,7 +36,6 @@ CONN_SEG_VIS, ) from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py index d9609413d..8a6f40959 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py @@ -7,6 +7,7 @@ from torch.optim.lr_scheduler import LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback from vis4d.engine.connectors import ( @@ -35,7 +36,6 @@ CONN_SEG_VIS, ) from vis4d.zoo.base.datasets.shift import get_shift_sem_seg_config -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/vit/vit_small_imagenet.py b/vis4d/zoo/vit/vit_small_imagenet.py index b19a3608d..85ae78f02 100644 --- a/vis4d/zoo/vit/vit_small_imagenet.py +++ b/vis4d/zoo/vit/vit_small_imagenet.py @@ -8,6 +8,7 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.engine.callbacks import EMACallback, EvaluatorCallback from vis4d.engine.connectors import ( CallbackConnector, @@ -34,7 +35,6 @@ CONN_IMAGENET_CLS_EVAL, get_imagenet_cls_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/vit/vit_tiny_imagenet.py b/vis4d/zoo/vit/vit_tiny_imagenet.py index ada082059..0d7f8c588 100644 --- a/vis4d/zoo/vit/vit_tiny_imagenet.py +++ b/vis4d/zoo/vit/vit_tiny_imagenet.py @@ -8,6 +8,7 @@ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.engine.callbacks import EMACallback, EvaluatorCallback from vis4d.engine.connectors import ( CallbackConnector, @@ -34,7 +35,6 @@ CONN_IMAGENET_CLS_EVAL, get_imagenet_cls_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: diff --git a/vis4d/zoo/yolox/data.py b/vis4d/zoo/yolox/data.py index dde2e30cd..eefda5e35 100644 --- a/vis4d/zoo/yolox/data.py +++ b/vis4d/zoo/yolox/data.py @@ -7,6 +7,7 @@ from ml_collections import ConfigDict from vis4d.config import class_config +from vis4d.config.typing import DataConfig from vis4d.data.const import CommonKeys as K from vis4d.data.data_pipe import DataPipe, MultiSampleDataPipe from vis4d.data.datasets.coco import COCO @@ -41,7 +42,6 @@ from vis4d.engine.connectors import data_key, pred_key from vis4d.zoo.base import get_inference_dataloaders_cfg from vis4d.zoo.base.callable import get_callable_cfg -from vis4d.zoo.typing import DataConfig CONN_COCO_BBOX_EVAL = { "coco_image_id": data_key(K.sample_names), diff --git a/vis4d/zoo/yolox/yolox_s_300e_coco.py b/vis4d/zoo/yolox/yolox_s_300e_coco.py index c47a49171..7403d3625 100644 --- a/vis4d/zoo/yolox/yolox_s_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_s_300e_coco.py @@ -6,6 +6,7 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -23,7 +24,6 @@ get_yolox_cfg, get_yolox_optimizers_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters from vis4d.zoo.yolox.data import CONN_COCO_BBOX_EVAL, get_coco_yolox_cfg CONN_BBOX_2D_TRAIN = {"images": K.images} diff --git a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py index 83dcc36f0..e1c8d50f3 100644 --- a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py @@ -6,6 +6,7 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.const import CommonKeys as K from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -23,7 +24,6 @@ get_yolox_cfg, get_yolox_optimizers_cfg, ) -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters from vis4d.zoo.yolox.data import CONN_COCO_BBOX_EVAL, get_coco_yolox_cfg CONN_BBOX_2D_TRAIN = {"images": K.images} From 6daa7a2933a787ab88357c82568e4fb282cf39b0 Mon Sep 17 00:00:00 2001 From: Thomas Huang Date: Tue, 23 Apr 2024 14:02:16 +0200 Subject: [PATCH 05/12] Updating QDTrack zoo --- tests/vis4d-test-data | 2 +- tests/zoo/qdtrack_test.py | 4 +- vis4d/zoo/qdtrack/README.md | 6 +- vis4d/zoo/qdtrack/__init__.py | 4 +- .../qdtrack/qdtrack_yolox_x_25e_bdd100k.py | 168 ++++++++++++++++++ 5 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index 031f308f1..57a6934ed 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit 031f308f10742dd083ec681a9ea7b7568744cd01 +Subproject commit 57a6934ed349d42c47208d175380084936c2204b diff --git a/tests/zoo/qdtrack_test.py b/tests/zoo/qdtrack_test.py index 6861d0a9a..a1922620c 100644 --- a/tests/zoo/qdtrack_test.py +++ b/tests/zoo/qdtrack_test.py @@ -30,7 +30,7 @@ def test_yolox_x_50e_bdd100k(self) -> None: This instantiates the config and compares it to a ground truth. """ compare_configs( - f"{self.config_prefix}.qdtrack_yolox_x_50e_bdd100k", - f"{self.gt_config_path}/qdtrack_yolox_x_50e_bdd100k.yaml", + f"{self.config_prefix}.qdtrack_yolox_x_25e_bdd100k", + f"{self.gt_config_path}/qdtrack_yolox_x_25e_bdd100k.yaml", self.varying_keys, ) diff --git a/vis4d/zoo/qdtrack/README.md b/vis4d/zoo/qdtrack/README.md index b78496db6..386ab822b 100644 --- a/vis4d/zoo/qdtrack/README.md +++ b/vis4d/zoo/qdtrack/README.md @@ -15,7 +15,7 @@ Similarity learning has been recognized as a crucial step for object tracking. H | Detector | Base Network | Strong Augs. | mMOTA-val | mIDF1-val | ID Sw.-val | Config | Weights | Visuals | | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | | Faster R-CNN | R-50-FPN | ✓ | 37.7 | 52.7 | 7257| [config](./qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py) | [model](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k_456b1e.pt) | [visuals](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k_vis.zip) | -| YOLOX-x | CSPNet | ✓ | 42.3 | 55.1 | 9164 | [config](./qdtrack/qdtrack_yolox_x_50e_bdd100k.py) | [model](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_yolox_x_25e_bdd100k_c14af2.pt) | [visuals](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_yolox_x_25e_bdd100k_vis.zip) | +| YOLOX-x | CSPNet | ✓ | 42.3 | 55.1 | 9164 | [config](./qdtrack/qdtrack_yolox_x_25e_bdd100k.py) | [model](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_yolox_x_25e_bdd100k_c14af2.pt) | [visuals](https://dl.cv.ethz.ch/vis4d/qdtrack/qdtrack_yolox_x_25e_bdd100k_vis.zip) | ## Getting Started @@ -25,7 +25,7 @@ Similarity learning has been recognized as a crucial step for object tracking. H python -m vis4d.pl fit --config vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py --gpus 8 # YOLOX -python -m vis4d.pl fit --config vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py --gpus 8 +python -m vis4d.pl fit --config vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py --gpus 8 ``` ### Inference @@ -34,7 +34,7 @@ python -m vis4d.pl fit --config vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py python -m vis4d.pl test --config vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py --ckpt ${checkpoint_path} --gpus ${num_gpus} # YOLOX -python -m vis4d.pl test --config vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py --ckpt ${checkpoint_path} --gpus ${num_gpus} +python -m vis4d.pl test --config vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py --ckpt ${checkpoint_path} --gpus ${num_gpus} ``` ## Citation diff --git a/vis4d/zoo/qdtrack/__init__.py b/vis4d/zoo/qdtrack/__init__.py index c8b6ff430..8d58127bb 100644 --- a/vis4d/zoo/qdtrack/__init__.py +++ b/vis4d/zoo/qdtrack/__init__.py @@ -2,7 +2,7 @@ from . import ( qdtrack_frcnn_r50_fpn_augs_1x_bdd100k, - qdtrack_yolox_x_50e_bdd100k, + qdtrack_yolox_x_25e_bdd100k, ) # Lists of available models in BDD100K Model Zoo. @@ -10,5 +10,5 @@ "qdtrack_frcnn_r50_fpn_augs_1x_bdd100k": ( qdtrack_frcnn_r50_fpn_augs_1x_bdd100k ), - "qdtrack_yolox_x_50e_bdd100k": qdtrack_yolox_x_50e_bdd100k, + "qdtrack_yolox_x_25e_bdd100k": qdtrack_yolox_x_25e_bdd100k, } diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py new file mode 100644 index 000000000..14723aa7f --- /dev/null +++ b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py @@ -0,0 +1,168 @@ +# pylint: disable=duplicate-code +"""QDTrack with YOLOX-x on BDD100K.""" +from __future__ import annotations + +import pytorch_lightning as pl +from lightning.pytorch.callbacks import ModelCheckpoint + +from vis4d.config import class_config +from vis4d.data.datasets.bdd100k import bdd100k_track_map +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback +from vis4d.engine.connectors import CallbackConnector, DataConnector +from vis4d.eval.bdd100k import BDD100KTrackEvaluator +from vis4d.vis.image import BoundingBoxVisualizer +from vis4d.zoo.base import ( + get_default_callbacks_cfg, + get_default_cfg, + get_default_pl_trainer_cfg, +) +from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TRACK_VIS +from vis4d.zoo.base.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL +from vis4d.zoo.base.models.qdtrack import ( + CONN_BBOX_2D_TEST, + CONN_BBOX_2D_TRAIN, + get_qdtrack_yolox_cfg, +) +from vis4d.zoo.base.models.yolox import ( + get_yolox_callbacks_cfg, + get_yolox_optimizers_cfg, +) +from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg +from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters + + +def get_config() -> ExperimentConfig: + """Returns the config dict for qdtrack on bdd100k. + + Returns: + ExperimentConfig: The configuration + """ + ###################################################### + ## General Config ## + ###################################################### + config = get_default_cfg(exp_name="qdtrack_yolox_x_25e_bdd100k") + config.checkpoint_period = 5 + config.check_val_every_n_epoch = 5 + + # Hyper Parameters + params = ExperimentParameters() + params.samples_per_gpu = 8 # batch size = 8 GPUs * 8 samples per GPU = 64 + params.workers_per_gpu = 8 + params.lr = 0.001 + params.num_epochs = 25 + config.params = params + + ###################################################### + ## Datasets with augmentations ## + ###################################################### + data_backend = class_config(HDF5Backend) + + config.data = get_bdd100k_track_cfg( + data_backend=data_backend, + samples_per_gpu=params.samples_per_gpu, + workers_per_gpu=params.workers_per_gpu, + ) + + ###################################################### + ## MODEL ## + ###################################################### + num_classes = len(bdd100k_track_map) + weights = ( + "mmdet://yolox/yolox_x_8x8_300e_coco/" + "yolox_x_8x8_300e_coco_20211126_140254-1ef88d67.pth" + ) + config.model, config.loss = get_qdtrack_yolox_cfg( + num_classes, "xlarge", weights=weights + ) + + ###################################################### + ## OPTIMIZERS ## + ###################################################### + # we use a schedule with 50 epochs, but only train for 25 epochs + num_total_epochs, num_last_epochs = 50, 10 + config.optimizers = get_yolox_optimizers_cfg( + params.lr, num_total_epochs, 1, num_last_epochs + ) + + ###################################################### + ## DATA CONNECTOR ## + ###################################################### + config.train_data_connector = class_config( + DataConnector, key_mapping=CONN_BBOX_2D_TRAIN + ) + + config.test_data_connector = class_config( + DataConnector, key_mapping=CONN_BBOX_2D_TEST + ) + + ###################################################### + ## CALLBACKS ## + ###################################################### + # Logger and Checkpoint + callbacks = get_default_callbacks_cfg( + config.output_dir, refresh_rate=config.log_every_n_steps + ) + + # YOLOX callbacks + callbacks += get_yolox_callbacks_cfg( + switch_epoch=num_total_epochs - num_last_epochs, num_sizes=0 + ) + + # Visualizer + callbacks.append( + class_config( + VisualizerCallback, + visualizer=class_config( + BoundingBoxVisualizer, vis_freq=500, image_mode="BGR" + ), + save_prefix=config.output_dir, + test_connector=class_config( + CallbackConnector, key_mapping=CONN_BBOX_2D_TRACK_VIS + ), + ) + ) + + # Evaluator + callbacks.append( + class_config( + EvaluatorCallback, + evaluator=class_config( + BDD100KTrackEvaluator, + annotation_path="data/bdd100k/labels/box_track_20/val/", + ), + test_connector=class_config( + CallbackConnector, key_mapping=CONN_BDD100K_TRACK_EVAL + ), + ) + ) + + config.callbacks = callbacks + + ###################################################### + ## PL CLI ## + ###################################################### + # PL Trainer args + pl_trainer = get_default_pl_trainer_cfg(config) + pl_trainer.max_epochs = params.num_epochs + pl_trainer.check_val_every_n_epoch = config.check_val_every_n_epoch + pl_trainer.checkpoint_callback = class_config( + ModelCheckpoint, + dirpath=config.get_ref("output_dir") + "/checkpoints", + verbose=True, + save_last=True, + save_on_train_epoch_end=True, + every_n_epochs=config.checkpoint_period, + save_top_k=5, + mode="max", + monitor="step", + ) + pl_trainer.wandb = True + pl_trainer.precision = "16-mixed" + config.pl_trainer = pl_trainer + + # PL Callbacks + pl_callbacks: list[pl.callbacks.Callback] = [] + config.pl_callbacks = pl_callbacks + + return config.value_mode() From db9fb556339be400ed94181acee5633decdb68c9 Mon Sep 17 00:00:00 2001 From: Thomas Huang Date: Tue, 23 Apr 2024 14:02:59 +0200 Subject: [PATCH 06/12] Remove old QDTrack config --- .../qdtrack/qdtrack_yolox_x_50e_bdd100k.py | 168 ------------------ 1 file changed, 168 deletions(-) delete mode 100644 vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py deleted file mode 100644 index 60454d369..000000000 --- a/vis4d/zoo/qdtrack/qdtrack_yolox_x_50e_bdd100k.py +++ /dev/null @@ -1,168 +0,0 @@ -# pylint: disable=duplicate-code -"""QDTrack with YOLOX-x on BDD100K.""" -from __future__ import annotations - -import pytorch_lightning as pl -from lightning.pytorch.callbacks import ModelCheckpoint - -from vis4d.config import class_config -from vis4d.config.typing import ExperimentConfig, ExperimentParameters -from vis4d.data.datasets.bdd100k import bdd100k_track_map -from vis4d.data.io.hdf5 import HDF5Backend -from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback -from vis4d.engine.connectors import CallbackConnector, DataConnector -from vis4d.eval.bdd100k import BDD100KTrackEvaluator -from vis4d.vis.image import BoundingBoxVisualizer -from vis4d.zoo.base import ( - get_default_callbacks_cfg, - get_default_cfg, - get_default_pl_trainer_cfg, -) -from vis4d.zoo.base.data_connectors import CONN_BBOX_2D_TRACK_VIS -from vis4d.zoo.base.datasets.bdd100k import CONN_BDD100K_TRACK_EVAL -from vis4d.zoo.base.models.qdtrack import ( - CONN_BBOX_2D_TEST, - CONN_BBOX_2D_TRAIN, - get_qdtrack_yolox_cfg, -) -from vis4d.zoo.base.models.yolox import ( - get_yolox_callbacks_cfg, - get_yolox_optimizers_cfg, -) -from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg - - -def get_config() -> ExperimentConfig: - """Returns the config dict for qdtrack on bdd100k. - - Returns: - ExperimentConfig: The configuration - """ - ###################################################### - ## General Config ## - ###################################################### - config = get_default_cfg(exp_name="qdtrack_yolox_x_50e_bdd100k") - config.checkpoint_period = 5 - config.check_val_every_n_epoch = 5 - - # Hyper Parameters - params = ExperimentParameters() - params.samples_per_gpu = 8 # batch size = 8 GPUs * 8 samples per GPU = 64 - params.workers_per_gpu = 8 - params.lr = 0.001 - params.num_epochs = 25 - config.params = params - - ###################################################### - ## Datasets with augmentations ## - ###################################################### - data_backend = class_config(HDF5Backend) - - config.data = get_bdd100k_track_cfg( - data_backend=data_backend, - samples_per_gpu=params.samples_per_gpu, - workers_per_gpu=params.workers_per_gpu, - ) - - ###################################################### - ## MODEL ## - ###################################################### - num_classes = len(bdd100k_track_map) - weights = ( - "mmdet://yolox/yolox_x_8x8_300e_coco/" - "yolox_x_8x8_300e_coco_20211126_140254-1ef88d67.pth" - ) - config.model, config.loss = get_qdtrack_yolox_cfg( - num_classes, "xlarge", weights=weights - ) - - ###################################################### - ## OPTIMIZERS ## - ###################################################### - # we use a schedule with 50 epochs, but only train for 25 epochs - num_total_epochs, num_last_epochs = 50, 10 - config.optimizers = get_yolox_optimizers_cfg( - params.lr, num_total_epochs, 1, num_last_epochs - ) - - ###################################################### - ## DATA CONNECTOR ## - ###################################################### - config.train_data_connector = class_config( - DataConnector, key_mapping=CONN_BBOX_2D_TRAIN - ) - - config.test_data_connector = class_config( - DataConnector, key_mapping=CONN_BBOX_2D_TEST - ) - - ###################################################### - ## CALLBACKS ## - ###################################################### - # Logger and Checkpoint - callbacks = get_default_callbacks_cfg( - config.output_dir, refresh_rate=config.log_every_n_steps - ) - - # YOLOX callbacks - callbacks += get_yolox_callbacks_cfg( - switch_epoch=num_total_epochs - num_last_epochs, num_sizes=0 - ) - - # Visualizer - callbacks.append( - class_config( - VisualizerCallback, - visualizer=class_config( - BoundingBoxVisualizer, vis_freq=500, image_mode="BGR" - ), - save_prefix=config.output_dir, - test_connector=class_config( - CallbackConnector, key_mapping=CONN_BBOX_2D_TRACK_VIS - ), - ) - ) - - # Evaluator - callbacks.append( - class_config( - EvaluatorCallback, - evaluator=class_config( - BDD100KTrackEvaluator, - annotation_path="data/bdd100k/labels/box_track_20/val/", - ), - test_connector=class_config( - CallbackConnector, key_mapping=CONN_BDD100K_TRACK_EVAL - ), - ) - ) - - config.callbacks = callbacks - - ###################################################### - ## PL CLI ## - ###################################################### - # PL Trainer args - pl_trainer = get_default_pl_trainer_cfg(config) - pl_trainer.max_epochs = params.num_epochs - pl_trainer.check_val_every_n_epoch = config.check_val_every_n_epoch - pl_trainer.checkpoint_callback = class_config( - ModelCheckpoint, - dirpath=config.get_ref("output_dir") + "/checkpoints", - verbose=True, - save_last=True, - save_on_train_epoch_end=True, - every_n_epochs=config.checkpoint_period, - save_top_k=5, - mode="max", - monitor="step", - ) - pl_trainer.wandb = True - pl_trainer.precision = "16-mixed" - config.pl_trainer = pl_trainer - - # PL Callbacks - pl_callbacks: list[pl.callbacks.Callback] = [] - config.pl_callbacks = pl_callbacks - - return config.value_mode() From 9c6f1ee2e8ade7ed72e8e5ae685e3bea80446d84 Mon Sep 17 00:00:00 2001 From: Thomas Huang Date: Tue, 23 Apr 2024 14:13:56 +0200 Subject: [PATCH 07/12] Fix minor issue in config --- vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py index 14723aa7f..0cfcbabd6 100644 --- a/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py @@ -6,6 +6,7 @@ from lightning.pytorch.callbacks import ModelCheckpoint from vis4d.config import class_config +from vis4d.config.typing import ExperimentConfig, ExperimentParameters from vis4d.data.datasets.bdd100k import bdd100k_track_map from vis4d.data.io.hdf5 import HDF5Backend from vis4d.engine.callbacks import EvaluatorCallback, VisualizerCallback @@ -29,7 +30,6 @@ get_yolox_optimizers_cfg, ) from vis4d.zoo.qdtrack.data_yolox import get_bdd100k_track_cfg -from vis4d.zoo.typing import ExperimentConfig, ExperimentParameters def get_config() -> ExperimentConfig: From 0c16633b7d9e0d516e0f1e6d2e5e758e0b6895fd Mon Sep 17 00:00:00 2001 From: Thomas Huang Date: Tue, 23 Apr 2024 18:39:08 +0200 Subject: [PATCH 08/12] Update test data --- tests/vis4d-test-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index 57a6934ed..a41d79370 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit 57a6934ed349d42c47208d175380084936c2204b +Subproject commit a41d7937098c13de726bf9f46a86343a38eeea83 From 1015b5de749a0db73462a80d869e7582c06ab4ec Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Wed, 8 May 2024 15:16:59 +0200 Subject: [PATCH 09/12] feat: Update readme and scripts. --- .gitignore | 4 -- README.md | 76 ++++++++++++++++++++++++------ docs/source/user_guide/install.rst | 4 +- scripts/install_gpu_conda.sh | 4 +- scripts/slurm_test.sh | 25 ---------- scripts/slurm_train.sh | 25 ---------- tests/vis4d-test-data | 2 +- vis4d/data/transforms/resize.py | 13 +++-- 8 files changed, 76 insertions(+), 77 deletions(-) delete mode 100644 scripts/slurm_test.sh delete mode 100644 scripts/slurm_train.sh diff --git a/.gitignore b/.gitignore index 13e021e9f..7b72005b9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,13 +43,9 @@ coverage/ # downloaded repos and models scalabel/bot/experimental/* -# python cache -__pycache__/ - # python virtual environment env/ - # vscode workspace configuration *.code-workspace diff --git a/README.md b/README.md index 188b7ed7e..d3b62110a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,12 @@ A modular library for 4D scene understanding

+## Quickstart + +You can checkout our [documentation](https://docs.vis.xyz/4d/index.html). + +You can use the [template](https://github.com/SysCV/vis4d-template) here to start your own project with Vis4D. + ## Installation Installation is as easy as @@ -26,20 +32,20 @@ python3 -m pip install vis4d ```bash # vis4d.engine -python -m vis4d fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 +vis4d fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 # vis4d.pl -python -m vis4d.pl fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 +vis4d-pl fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 ``` - To test a model ```bash # vis4d.engine -python -m vis4d test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 +vis4d test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 # vis4d.pl -python -m vis4d.pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 +vis4d-pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 ``` ## DDP @@ -53,18 +59,17 @@ python -m vis4d.pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpu ./scripts/dist_train.sh # vis4d.pl -python -m vis4d.pl fit --config --gpus +vis4d-pl fit --config --gpus ``` -- SLURM batch job. Need to config the submission file. +- SLURM ```bash # vis4d.engine -sbatch scripts/slurm_train.sh +srun vis4d fit --config --gpus --slurm True # vis4d.pl -srun --cpus-per-task=4 --gres=gpumem:20G python -m vis4d.pl.run fit \ - --config --gpus +srun vis4d-pl fit --config --gpus ``` ### Testing @@ -76,20 +81,61 @@ srun --cpus-per-task=4 --gres=gpumem:20G python -m vis4d.pl.run fit \ ./scripts/dist_test.sh # vis4d.pl -python -m vis4d.pl test --config --gpus +vis4d-pl test --config --gpus ``` -- SLURM batch job. Need to config the submission file. +- SLURM ```bash # vis4d.engine -sbatch scripts/slurm_test.sh +srun vis4d test --config --gpus --slurm True # vis4d.pl -srun --cpus-per-task=4 --gres=gpumem:20G python -m vis4d.pl.run test \ - --config --gpus +srun vis4d-pl test --config --gpus ``` -## Contribute +## Acknowledgement +Vis4D is a group effort by our team at ETH Zurich. +[Yung-Hsu Yang](https://royyang0714.github.io/) built the current version and will be the main maintainer of the codebase. + +Vis4D was originally written by [Tobias Fischer](https://tobiasfshr.github.io/) during the first three years of his Ph.D. at ETH Zurich, [Thomas E. Huang](https://www.thomasehuang.com/) helped contribute many models, [Tao Sun](https://www.suniique.com/) implemented the ViT models and designed the evaluation pipeline, and[René Zurbrügg](https://github.com/renezurbruegg) designed the config system. + + +## Contributors +**Project Leads** +- [Yung-Hsu Yang](https://royyang0714.github.io/)* +- [Tobias Fischer](https://tobiasfshr.github.io/)* + +**Core Contributors** +- [Thomas E. Huang](https://www.thomasehuang.com/) +- [Tao Sun](https://www.suniique.com/) +- [René Zurbrügg](https://github.com/renezurbruegg) + +**Advisors** +- [Fisher Yu](https://www.yf.io/) + +`*` denotes equal contribution + +**We are open to contributions and suggestions, feel free to reach out to us.** [Check out our contribution guidelines for this project](docs/source/dev_guide/contribute.rst) + +**Community Contributors** + + + + + + +## Citation + +If you find Vis4D is useful for your research, please consider citing the following BibTeX entry. + +```bibtex +@misc{vis4d_2024, + author = {{Yung-Hsu Yang and Tobias Fischer and Thomas E. Huang} and René Zurbrügg and Tao Sun and Fisher Yu}, + title = {Vis4D}, + howpublished = {\url{https://github.com/SysCV/vis4d}}, + year = {2024} +} +``` diff --git a/docs/source/user_guide/install.rst b/docs/source/user_guide/install.rst index 4ddfbf1f3..219fc8d52 100644 --- a/docs/source/user_guide/install.rst +++ b/docs/source/user_guide/install.rst @@ -5,7 +5,7 @@ Installation ======= Package ======= -We currently support Python 3.9+ and PyTorch 1.10.0+. +We currently support Python 3.10+ and PyTorch 2.0.0+. We recommand to install in a new virtual environment, e.g. conda or virtualenv. @@ -30,7 +30,7 @@ If you want to build the package from source and specify CUDA version, you can c git clone https://github.com/SysCV/vis4d.git cd vis4d - python3 -m pip install -r requirements/install.txt -f https://download.pytorch.org/whl/cu117/torch_stable.html + python3 -m pip install -r requirements/install.txt -f https://download.pytorch.org/whl/cu118/torch_stable.html python3 -m pip install -r requirements/torch-lib.txt python3 -m pip install -e . diff --git a/scripts/install_gpu_conda.sh b/scripts/install_gpu_conda.sh index a84ad0ab2..a6ae8be0a 100644 --- a/scripts/install_gpu_conda.sh +++ b/scripts/install_gpu_conda.sh @@ -4,7 +4,7 @@ source $CONDA_PATH/etc/profile.d/conda.sh if [ -z "$1" ] then - CUDA_VERSION=11.7 + CUDA_VERSION=11.8 else CUDA_VERSION=$1 fi @@ -14,4 +14,4 @@ conda activate vis4d-dev conda install cudatoolkit=$CUDA_VERSION -c pytorch -c nvidia python3 -m pip install -r requirements/install.txt -f https://download.pytorch.org/whl/cu${CUDA_VERSION//.}/torch_stable.html python3 -m pip install -r requirements/torch-lib.txt -python3 setup.py develop +pip install -e . diff --git a/scripts/slurm_test.sh b/scripts/slurm_test.sh deleted file mode 100644 index beab7fae6..000000000 --- a/scripts/slurm_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=Vis4D # Job name -#SBATCH --nodes=1 # Numbers of node -#SBATCH --ntasks=8 # Numbers of task -#SBATCH --ntasks-per-node=8 # Tasks per node -#SBATCH --cpus-per-task=4 # Cores of cpu per task -#SBATCH --mem-per-cpu=4G # Memory per core -#SBATCH --gpus-per-node=8 # Numbers of gpu per node -#SBATCH --gres=gpumem:20G # Memory per gpu -#SBATCH --time=24:00:00 # 4h / 24h / 120h -#SBATCH --tmp=100G # Local scratch - -module load gcc/8.2.0 -module load zlib/1.2.9 -module load python/3.10.4 -module load cuda/11.7.0 - -export PYTHONPATH= -source /cluster/home/$USER/vis4d/bin/activate - -srun --cpus-per-task=4 --gres=gpumem:20G --kill-on-bad-exit=1 \ - python -m vis4d.engine.run test \ - --config configs/faster_rcnn/faster_rcnn_coco.py \ - --gpus 8 \ - --slurm True diff --git a/scripts/slurm_train.sh b/scripts/slurm_train.sh deleted file mode 100644 index d8459fece..000000000 --- a/scripts/slurm_train.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=Vis4D # Job name -#SBATCH --nodes=1 # Numbers of node -#SBATCH --ntasks=8 # Numbers of task -#SBATCH --ntasks-per-node=8 # Tasks per node -#SBATCH --cpus-per-task=4 # Cores of cpu per task -#SBATCH --mem-per-cpu=4G # Memory per core -#SBATCH --gpus-per-node=8 # Numbers of gpu per node -#SBATCH --gres=gpumem:20G # Memory per gpu -#SBATCH --time=24:00:00 # 4h / 24h / 120h -#SBATCH --tmp=100G # Local scratch - -module load gcc/8.2.0 -module load zlib/1.2.9 -module load python/3.10.4 -module load cuda/11.7.0 - -export PYTHONPATH= -source /cluster/home/$USER/vis4d/bin/activate - -srun --cpus-per-task=4 --gres=gpumem:20G --kill-on-bad-exit=1 \ - python -m vis4d.engine.run fit \ - --config configs/faster_rcnn/faster_rcnn_coco.py \ - --gpus 8 \ - --slurm True diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index a41d79370..1e52c1948 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit a41d7937098c13de726bf9f46a86343a38eeea83 +Subproject commit 1e52c194859cfc09ee2f10af595303da3646d7d3 diff --git a/vis4d/data/transforms/resize.py b/vis4d/data/transforms/resize.py index 4c4eff1ae..013ff2bcb 100644 --- a/vis4d/data/transforms/resize.py +++ b/vis4d/data/transforms/resize.py @@ -209,6 +209,7 @@ def __init__( self, interpolation: str = "nearest", rescale_depth_values: bool = False, + check_scale_factors: bool = False, ): """Initialize the transform. @@ -219,9 +220,14 @@ def __init__( be rescaled according to the new scale factor. Defaults to False. This is useful if we want to keep the intrinsic parameters of the camera the same. + check_scale_factors (bool, optional): If the scale factors should + be checked to ensure they are the same. Defaults to False. + If False, the scale factor is assumed to be the same for both + dimensions and will just use the first scale factor. """ self.interpolation = interpolation self.rescale_depth_values = rescale_depth_values + self.check_scale_factors = check_scale_factors def __call__( self, @@ -245,9 +251,10 @@ def __call__( .squeeze(0) ) if self.rescale_depth_values: - assert np.isclose( - scale_factor[0], scale_factor[1], atol=1e-4 - ), "Depth map scale factors must be the same" + if self.check_scale_factors: + assert np.isclose( + scale_factor[0], scale_factor[1], atol=1e-4 + ), "Depth map scale factors must be the same" depth_map_ /= scale_factor[0] depth_maps[i] = depth_map_.numpy() return depth_maps From 853602c1027221694b278ea618db8aa726c21541 Mon Sep 17 00:00:00 2001 From: zrene Date: Sat, 11 May 2024 12:39:53 +0200 Subject: [PATCH 10/12] update page --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3b62110a..9f6d805a6 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Vis4D was originally written by [Tobias Fischer](https://tobiasfshr.github.io/) **Core Contributors** - [Thomas E. Huang](https://www.thomasehuang.com/) - [Tao Sun](https://www.suniique.com/) -- [René Zurbrügg](https://github.com/renezurbruegg) +- [René Zurbrügg](https://renezurbruegg.github.io/) **Advisors** - [Fisher Yu](https://www.yf.io/) From 319bf980da9d9d9ec789230efc36d8680d903d49 Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Wed, 15 May 2024 15:54:40 +0200 Subject: [PATCH 11/12] feat: Update pyproject and fix new linting. --- docs/source/user_guide/install.rst | 11 +++++++- pyproject.toml | 6 ++-- requirements/install.txt | 4 +-- tests/data/datasets/scalabel_test.py | 2 ++ .../pointcloud/pointcloud_visualizer_test.py | 2 ++ vis4d/__init__.py | 2 ++ vis4d/common/ckpt.py | 2 +- vis4d/config/show_connection.py | 2 +- vis4d/data/datasets/bdd100k.py | 2 ++ vis4d/data/datasets/nuscenes.py | 2 ++ vis4d/data/datasets/nuscenes_mono.py | 2 ++ vis4d/data/datasets/nuscenes_trajectory.py | 12 +++++--- vis4d/data/datasets/scalabel.py | 6 ++++ vis4d/data/datasets/shift.py | 2 ++ vis4d/data/datasets/util.py | 2 ++ vis4d/data/io/hdf5.py | 2 +- vis4d/data/transforms/affine.py | 10 +++++-- vis4d/data/transforms/autoaugment.py | 2 ++ vis4d/data/transforms/photometric.py | 2 ++ vis4d/data/transforms/resize.py | 4 ++- vis4d/engine/experiment.py | 9 ++++-- vis4d/engine/optim/scheduler.py | 5 ++-- vis4d/engine/run.py | 2 +- vis4d/eval/bdd100k/detect.py | 6 ++-- vis4d/eval/bdd100k/seg.py | 2 ++ vis4d/eval/bdd100k/track.py | 2 ++ vis4d/eval/coco/detect.py | 2 +- vis4d/eval/nuscenes/detect3d.py | 2 ++ vis4d/eval/scalabel/base.py | 2 ++ vis4d/eval/scalabel/detect.py | 22 +++++++++------ vis4d/eval/scalabel/track.py | 22 +++++++++------ vis4d/eval/shift/multitask_writer.py | 28 +++++++++++-------- vis4d/op/base/pointnet.py | 2 +- .../bevformer/spatial_cross_attention.py | 8 ++---- vis4d/op/detect3d/util.py | 4 ++- vis4d/op/geometry/rotation.py | 5 ++-- vis4d/op/layer/ms_deform_attn.py | 2 ++ vis4d/op/mask/util.py | 7 +++-- vis4d/pl/run.py | 2 +- vis4d/pl/training_module.py | 9 +++++- vis4d/zoo/base/models/qdtrack.py | 2 ++ vis4d/zoo/base/models/yolox.py | 2 ++ vis4d/zoo/bevformer/data.py | 2 ++ vis4d/zoo/run.py | 2 +- 44 files changed, 161 insertions(+), 69 deletions(-) diff --git a/docs/source/user_guide/install.rst b/docs/source/user_guide/install.rst index 219fc8d52..417f177e4 100644 --- a/docs/source/user_guide/install.rst +++ b/docs/source/user_guide/install.rst @@ -31,7 +31,6 @@ If you want to build the package from source and specify CUDA version, you can c cd vis4d python3 -m pip install -r requirements/install.txt -f https://download.pytorch.org/whl/cu118/torch_stable.html - python3 -m pip install -r requirements/torch-lib.txt python3 -m pip install -e . More information about torch and pytorch-lightning installation @@ -39,6 +38,16 @@ More information about torch and pytorch-lightning installation - `PyTorch `_ - `PyTorch Lightning `_ +----------------------- +Install CUDA Operations +----------------------- + +Some functionalities in the library require CUDA operations. You can install them by running: + +.. code:: bash + + python3 -m pip install -r requirements/torch-lib.txt + ================ Directory Layout ================ diff --git a/pyproject.toml b/pyproject.toml index f12accf20..8cc162d41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,12 +112,12 @@ plugins = ["numpy.typing.mypy_plugin"] [project] name = "vis4d" -version = "0.0" +version = "0.1.1" authors = [{name = "VIS @ ETH", email = "i@yf.io"}] description = "Vis4D Python package for Visual 4D scene understanding" readme = "README.md" license = { text="Apache 2.0"} -requires-python = ">=3.9.0" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", @@ -135,7 +135,7 @@ dependencies = {file = ["requirements/install.txt"]} include = ["vis4d*"] [project.urls] -"Documentation" = "https://vis.xyz/4d/" +"Documentation" = "https://docs.vis.xyz/4d" "Source" = "https://github.com/syscv/vis4d" "Tracker" = "https://github.com/syscv/" diff --git a/requirements/install.txt b/requirements/install.txt index 9c1c56610..c1f6cfe52 100644 --- a/requirements/install.txt +++ b/requirements/install.txt @@ -19,8 +19,8 @@ tensorboard termcolor terminaltables timm>=0.6.0 -torch>=1.10 -torchvision>=0.11 +torch>=2.0.0 +torchvision>=0.15.1 tqdm utm wheel diff --git a/tests/data/datasets/scalabel_test.py b/tests/data/datasets/scalabel_test.py index a0b87d693..712e378a1 100644 --- a/tests/data/datasets/scalabel_test.py +++ b/tests/data/datasets/scalabel_test.py @@ -11,6 +11,8 @@ if SCALABEL_AVAILABLE: from scalabel.label.typing import Config +else: + raise ImportError("scalabel is not available") IMAGE_INDICES = np.array([0, 1, 232875, 465749]) IMAGE_VALUES = np.array( diff --git a/tests/vis/pointcloud/pointcloud_visualizer_test.py b/tests/vis/pointcloud/pointcloud_visualizer_test.py index d027c9f86..5eab5c9cd 100644 --- a/tests/vis/pointcloud/pointcloud_visualizer_test.py +++ b/tests/vis/pointcloud/pointcloud_visualizer_test.py @@ -26,6 +26,8 @@ if OPEN3D_AVAILABLE: import open3d as o3d +else: + raise ImportError("open3d is not available") class TestPointcloudViewer(unittest.TestCase): diff --git a/vis4d/__init__.py b/vis4d/__init__.py index f5569352a..c407efa37 100644 --- a/vis4d/__init__.py +++ b/vis4d/__init__.py @@ -7,6 +7,8 @@ import logging +__version__ = "0.1.1" + _root_logger = logging.getLogger() _logger = logging.getLogger(__name__) _logger.setLevel(logging.INFO) diff --git a/vis4d/common/ckpt.py b/vis4d/common/ckpt.py index 0db727c16..fdd5b6e4a 100644 --- a/vis4d/common/ckpt.py +++ b/vis4d/common/ckpt.py @@ -233,7 +233,7 @@ def load_from_http( synchronize() if rank > 0: checkpoint = load_url(filename, map_location=map_location) - return checkpoint + return checkpoint # pylint: disable=used-before-assignment def get_torchvision_models() -> dict[str, str]: diff --git a/vis4d/config/show_connection.py b/vis4d/config/show_connection.py index 8bb793b75..e348815f9 100644 --- a/vis4d/config/show_connection.py +++ b/vis4d/config/show_connection.py @@ -5,7 +5,7 @@ import inspect from typing import Any, TypedDict, get_type_hints -from absl import app +from absl import app # pylint: disable=no-name-in-module from torch import nn from vis4d.common.typing import ArgsType diff --git a/vis4d/data/datasets/bdd100k.py b/vis4d/data/datasets/bdd100k.py index e978d1f4c..e3d2c1919 100644 --- a/vis4d/data/datasets/bdd100k.py +++ b/vis4d/data/datasets/bdd100k.py @@ -95,6 +95,8 @@ from bdd100k.label.to_scalabel import bdd100k_to_scalabel from scalabel.label.io import load from scalabel.label.typing import Dataset as ScalabelData +else: + raise ImportError("bdd100k or scalabel is not installed.") class BDD100K(Scalabel): diff --git a/vis4d/data/datasets/nuscenes.py b/vis4d/data/datasets/nuscenes.py index eda37a15e..a80270bfc 100644 --- a/vis4d/data/datasets/nuscenes.py +++ b/vis4d/data/datasets/nuscenes.py @@ -48,6 +48,8 @@ view_points, ) from nuscenes.utils.splits import create_splits_scenes +else: + raise ImportError("nusenes-devkit is not available.") nuscenes_class_map = { "bicycle": 0, diff --git a/vis4d/data/datasets/nuscenes_mono.py b/vis4d/data/datasets/nuscenes_mono.py index 19128eb66..9a48f9222 100644 --- a/vis4d/data/datasets/nuscenes_mono.py +++ b/vis4d/data/datasets/nuscenes_mono.py @@ -19,6 +19,8 @@ if NUSCENES_AVAILABLE: from nuscenes import NuScenes as NuScenesDevkit from nuscenes.utils.splits import create_splits_scenes +else: + raise ImportError("nusenes-devkit is not available.") class NuScenesMono(NuScenes): diff --git a/vis4d/data/datasets/nuscenes_trajectory.py b/vis4d/data/datasets/nuscenes_trajectory.py index 8da3c88ae..a155ac837 100644 --- a/vis4d/data/datasets/nuscenes_trajectory.py +++ b/vis4d/data/datasets/nuscenes_trajectory.py @@ -21,6 +21,8 @@ from nuscenes.eval.detection.utils import category_to_detection_name from nuscenes.utils.data_classes import Quaternion from nuscenes.utils.splits import create_splits_scenes +else: + raise ImportError("nusenes-devkit is not available.") class NuScenesTrajectory(CacheMappingMixin, Dataset): @@ -104,10 +106,12 @@ def _match_gt_pred( preds_center = [ pred["translation"][:2] for pred in same_class_preds ] - distance_matrix = cdist( - gt_world[:, :2], - np.array(preds_center).reshape(-1, 2), - )[0] + distance_matrix = ( + cdist( # pylint: disable=unsubscriptable-object + gt_world[:, :2], + np.array(preds_center).reshape(-1, 2), + )[0] + ) if distance_matrix[distance_matrix.argmin()] <= 2: match_pred = same_class_preds[distance_matrix.argmin()] diff --git a/vis4d/data/datasets/scalabel.py b/vis4d/data/datasets/scalabel.py index b535d60df..215c07ffd 100644 --- a/vis4d/data/datasets/scalabel.py +++ b/vis4d/data/datasets/scalabel.py @@ -57,6 +57,8 @@ get_matrix_from_extrinsics, get_matrix_from_intrinsics, ) +else: + raise ImportError("scalabel is not installed.") def load_intrinsics(intrinsics: Intrinsics) -> NDArrayF32: @@ -614,6 +616,8 @@ def instance_masks_from_scalabel( bitmask: NDArrayUI8 = (bitmask_raw > 0).astype( # type: ignore bitmask_raw.dtype ) + else: + raise ValueError("No mask found in label.") bitmask_list.append(bitmask) if len(bitmask_list) == 0: # pragma: no cover return np.empty((0, 0, 0), dtype=np.uint8) @@ -680,6 +684,8 @@ def semantic_masks_from_scalabel( bitmask: NDArrayUI8 = (bitmask_raw > 0).astype( # type: ignore bitmask_raw.dtype ) + else: + raise ValueError("No mask found in label.") bitmask_list.append(bitmask) if bg_as_class: foreground = ( diff --git a/vis4d/data/datasets/shift.py b/vis4d/data/datasets/shift.py index f6db0c220..fd79e2fce 100644 --- a/vis4d/data/datasets/shift.py +++ b/vis4d/data/datasets/shift.py @@ -80,6 +80,8 @@ from scalabel.label.io import parse from scalabel.label.typing import Config from scalabel.label.typing import Dataset as ScalabelData +else: + raise ImportError("scalabel is not installed.") def _get_extension(backend: DataBackend) -> str: diff --git a/vis4d/data/datasets/util.py b/vis4d/data/datasets/util.py index 38486fa3d..b4ffbcb97 100644 --- a/vis4d/data/datasets/util.py +++ b/vis4d/data/datasets/util.py @@ -40,6 +40,8 @@ cvtColor, imdecode, ) +else: + raise ImportError("cv2 is not installed.") def im_decode( diff --git a/vis4d/data/io/hdf5.py b/vis4d/data/io/hdf5.py index cfa1f8630..151077700 100644 --- a/vis4d/data/io/hdf5.py +++ b/vis4d/data/io/hdf5.py @@ -21,7 +21,7 @@ import h5py from h5py import File else: - File = None # pylint: disable=invalid-name + raise ImportError("Please install h5py to enable HDF5Backend.") class HDF5Backend(DataBackend): diff --git a/vis4d/data/transforms/affine.py b/vis4d/data/transforms/affine.py index 0f6f5cc5d..b4c1bacc1 100644 --- a/vis4d/data/transforms/affine.py +++ b/vis4d/data/transforms/affine.py @@ -22,6 +22,8 @@ if OPENCV_AVAILABLE: import cv2 +else: + raise ImportError("Please install opencv-python to use this module.") class AffineParam(TypedDict): @@ -233,12 +235,16 @@ def __call__( zip(images, warp_matrix_list, height_list, width_list) ): image = image[0].astype(np.uint8) if self.as_int else image[0] - image = cv2.warpPerspective( # pylint: disable=no-member + image = cv2.warpPerspective( # pylint: disable=no-member, unsubscriptable-object, line-too-long image, warp_matrix, dsize=(width, height), borderValue=self.border_val, - )[None, ...].astype(np.float32) + )[ + None, ... + ].astype( + np.float32 + ) images[i] = image input_hw_list.append((height, width)) diff --git a/vis4d/data/transforms/autoaugment.py b/vis4d/data/transforms/autoaugment.py index 16b02ff67..f0bbd7812 100644 --- a/vis4d/data/transforms/autoaugment.py +++ b/vis4d/data/transforms/autoaugment.py @@ -22,6 +22,8 @@ auto_augment_policy, rand_augment_ops, ) +else: + raise ImportError("timm is not installed.") AugOp = Union[AutoAugment, RandAugment, AugMixAugment] diff --git a/vis4d/data/transforms/photometric.py b/vis4d/data/transforms/photometric.py index 63c39f522..0d19eb9a1 100644 --- a/vis4d/data/transforms/photometric.py +++ b/vis4d/data/transforms/photometric.py @@ -17,6 +17,8 @@ if OPENCV_AVAILABLE: import cv2 +else: + raise ImportError("cv2 is not installed.") @Transform(K.images, K.images) diff --git a/vis4d/data/transforms/resize.py b/vis4d/data/transforms/resize.py index 013ff2bcb..c613bc0e1 100644 --- a/vis4d/data/transforms/resize.py +++ b/vis4d/data/transforms/resize.py @@ -26,6 +26,8 @@ INTER_LINEAR, INTER_NEAREST, ) +else: + raise ImportError("Please install opencv-python to use this module.") class ResizeParam(TypedDict): @@ -407,7 +409,7 @@ def resize_image( "area": INTER_AREA, "lanczos": INTER_LANCZOS4, } - return cv2.resize( # pylint: disable=no-member + return cv2.resize( # pylint: disable=no-member, unsubscriptable-object inputs[0].astype(np.uint8), (shape[1], shape[0]), interpolation=cv2_interp_codes[interpolation], diff --git a/vis4d/engine/experiment.py b/vis4d/engine/experiment.py index 10cac2200..a0781ab12 100644 --- a/vis4d/engine/experiment.py +++ b/vis4d/engine/experiment.py @@ -198,6 +198,7 @@ def run_experiment( # Resume training if resume: + assert mode == "fit", "Resume is only supported in fit mode" if ckpt_path is None: ckpt_path = os.path.join( config.output_dir, "checkpoints/last.ckpt" @@ -210,10 +211,14 @@ def run_experiment( epoch = ckpt["epoch"] + 1 global_step = ckpt["global_step"] - for i, optimizer in enumerate(optimizers): + for i, optimizer in enumerate( + optimizers # pylint:disable=possibly-used-before-assignment + ): optimizer.load_state_dict(ckpt["optimizers"][i]) - for i, lr_scheduler in enumerate(lr_schedulers): + for i, lr_scheduler in enumerate( + lr_schedulers # pylint:disable=possibly-used-before-assignment + ): lr_scheduler.load_state_dict(ckpt["lr_schedulers"][i]) else: epoch = 0 diff --git a/vis4d/engine/optim/scheduler.py b/vis4d/engine/optim/scheduler.py index 674f4bf2f..2e1e63370 100644 --- a/vis4d/engine/optim/scheduler.py +++ b/vis4d/engine/optim/scheduler.py @@ -1,4 +1,5 @@ -"""Vis4D LR schedulers.""" +# pylint: disable=no-member +"""LR schedulers.""" from __future__ import annotations @@ -251,7 +252,7 @@ def get_lr(self) -> list[float]: # type: ignore return self.base_lrs factors = [ base_lr * (2 * step_count + 1) / self.max_steps**2 - for base_lr in self.base_lrs + for base_lr in self.base_lrs # pylint: disable=not-an-iterable ] if step_count == 0: return factors diff --git a/vis4d/engine/run.py b/vis4d/engine/run.py index 406ab3cb0..c0033ab4d 100644 --- a/vis4d/engine/run.py +++ b/vis4d/engine/run.py @@ -2,7 +2,7 @@ from __future__ import annotations -from absl import app +from absl import app # pylint: disable=no-name-in-module from vis4d.common import ArgsType from vis4d.common.logging import rank_zero_info diff --git a/vis4d/eval/bdd100k/detect.py b/vis4d/eval/bdd100k/detect.py index 497e62d19..a252c9188 100644 --- a/vis4d/eval/bdd100k/detect.py +++ b/vis4d/eval/bdd100k/detect.py @@ -2,11 +2,13 @@ from __future__ import annotations -from vis4d.common.imports import BDD100K_AVAILABLE, SCALABEL_AVAILABLE +from vis4d.common.imports import BDD100K_AVAILABLE from vis4d.eval.scalabel import ScalabelDetectEvaluator -if BDD100K_AVAILABLE and SCALABEL_AVAILABLE: +if BDD100K_AVAILABLE: from bdd100k.common.utils import load_bdd100k_config +else: + raise ImportError("bdd100k is not installed.") class BDD100KDetectEvaluator(ScalabelDetectEvaluator): diff --git a/vis4d/eval/bdd100k/seg.py b/vis4d/eval/bdd100k/seg.py index 776e54f35..d445bda3e 100644 --- a/vis4d/eval/bdd100k/seg.py +++ b/vis4d/eval/bdd100k/seg.py @@ -22,6 +22,8 @@ from scalabel.label.io import load from scalabel.label.transforms import mask_to_rle from scalabel.label.typing import Frame, Label +else: + raise ImportError("scalabel or bdd100k is not installed.") class BDD100KSegEvaluator(Evaluator): diff --git a/vis4d/eval/bdd100k/track.py b/vis4d/eval/bdd100k/track.py index 6e4c7b5b7..a71b08f1f 100644 --- a/vis4d/eval/bdd100k/track.py +++ b/vis4d/eval/bdd100k/track.py @@ -14,6 +14,8 @@ from scalabel.eval.detect import evaluate_det from scalabel.eval.mot import acc_single_video_mot, evaluate_track from scalabel.label.io import group_and_sort +else: + raise ImportError("scalabel or bdd100k is not installed.") class BDD100KTrackEvaluator(ScalabelTrackEvaluator): diff --git a/vis4d/eval/coco/detect.py b/vis4d/eval/coco/detect.py index 676d1660f..0acc42651 100644 --- a/vis4d/eval/coco/detect.py +++ b/vis4d/eval/coco/detect.py @@ -217,7 +217,7 @@ def evaluate(self, metric: str) -> tuple[MetricLogs, str]: if metric == self.METRIC_DET: iou_type = "bbox" _predictions = self._predictions - elif metric == self.METRIC_INS_SEG: + else: # remove bbox for segm evaluation so cocoapi will use mask # area instead of box area iou_type = "segm" diff --git a/vis4d/eval/nuscenes/detect3d.py b/vis4d/eval/nuscenes/detect3d.py index 5974b774a..178b9c3cb 100644 --- a/vis4d/eval/nuscenes/detect3d.py +++ b/vis4d/eval/nuscenes/detect3d.py @@ -25,6 +25,8 @@ from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.evaluate import NuScenesEval from nuscenes.utils.data_classes import Quaternion +else: + raise ImportError("nuscenes-devkit is not installed.") def _parse_high_level_metrics( diff --git a/vis4d/eval/scalabel/base.py b/vis4d/eval/scalabel/base.py index 31326c62e..61a3e1534 100644 --- a/vis4d/eval/scalabel/base.py +++ b/vis4d/eval/scalabel/base.py @@ -14,6 +14,8 @@ from scalabel.label.io import load from scalabel.label.typing import Config, Frame from scalabel.label.utils import get_leaf_categories +else: + raise ImportError("scalabel is not installed.") class ScalabelEvaluator(Evaluator): diff --git a/vis4d/eval/scalabel/detect.py b/vis4d/eval/scalabel/detect.py index 7be4f307d..dae776672 100644 --- a/vis4d/eval/scalabel/detect.py +++ b/vis4d/eval/scalabel/detect.py @@ -15,6 +15,8 @@ from scalabel.eval.ins_seg import evaluate_ins_seg from scalabel.label.transforms import mask_to_rle, xyxy_to_box2d from scalabel.label.typing import Config, Frame, Label +else: + raise ImportError("scalabel is not installed.") class ScalabelDetectEvaluator(ScalabelEvaluator): @@ -82,6 +84,16 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ zip(boxes, scores, class_ids) ): box2d = xyxy_to_box2d(*box.tolist()) + + if pred_masks: + rle = mask_to_rle( + (masks[label_id] > self.mask_threshold).astype( + np.uint8 + ) + ) + else: + rle = None + label = Label( id=str(label_id), box2d=box2d, @@ -91,15 +103,7 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ else str(class_id) ), score=float(score), - rle=( - mask_to_rle( - (masks[label_id] > self.mask_threshold).astype( - np.uint8 - ) - ) - if pred_masks - else None - ), + rle=rle, ) labels.append(label) frame = Frame( diff --git a/vis4d/eval/scalabel/track.py b/vis4d/eval/scalabel/track.py index e55b8b4f2..7222b1eb9 100644 --- a/vis4d/eval/scalabel/track.py +++ b/vis4d/eval/scalabel/track.py @@ -16,6 +16,8 @@ from scalabel.label.io import group_and_sort from scalabel.label.transforms import mask_to_rle, xyxy_to_box2d from scalabel.label.typing import Config, Frame, Label +else: + raise ImportError("scalabel is not installed.") class ScalabelTrackEvaluator(ScalabelEvaluator): @@ -88,6 +90,16 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ zip(boxes, scores, class_ids, track_ids) ): box2d = xyxy_to_box2d(*box.tolist()) + + if pred_masks: + rle = mask_to_rle( + (masks[label_id] > self.mask_threshold).astype( + np.uint8 + ) + ) + else: + rle = None + label = Label( box2d=box2d, category=( @@ -97,15 +109,7 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ ), score=float(score), id=str(int(track_id)), - rle=( - mask_to_rle( - (masks[label_id] > self.mask_threshold).astype( - np.uint8 - ) - ) - if pred_masks - else None - ), + rle=rle, ) labels.append(label) frame = Frame( diff --git a/vis4d/eval/shift/multitask_writer.py b/vis4d/eval/shift/multitask_writer.py index 00f8949a5..9d9187000 100644 --- a/vis4d/eval/shift/multitask_writer.py +++ b/vis4d/eval/shift/multitask_writer.py @@ -22,6 +22,8 @@ if SCALABEL_AVAILABLE: from scalabel.label.transforms import mask_to_rle, xyxy_to_box2d from scalabel.label.typing import Dataset, Frame, Label +else: + raise ImportError("scalabel is not installed.") class SHIFTMultitaskWriter(Evaluator): @@ -196,6 +198,18 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ pred_boxes2d_classes[i], ): box2d = xyxy_to_box2d(*box.tolist()) + if pred_instance_masks: + rle = mask_to_rle( + (masks[class_id] > 0.0).astype(np.uint8) + ) + else: + rle = None + + if pred_boxes2d_track_ids: + track_id = str(int(track_ids[0])) + else: + track_id = None + label = Label( box2d=box2d, category=( @@ -204,18 +218,8 @@ def process_batch( # type: ignore # pylint: disable=arguments-differ else str(class_id) ), score=float(score), - rle=( - mask_to_rle( - (masks[class_id] > 0.0).astype(np.uint8) - ) - if pred_instance_masks - else None - ), - id=( - str(int(track_ids[0])) - if pred_boxes2d_track_ids - else None - ), + rle=rle, + id=track_id, ) labels.append(label) frame = Frame( diff --git a/vis4d/op/base/pointnet.py b/vis4d/op/base/pointnet.py index 9f0c72625..67588cae9 100644 --- a/vis4d/op/base/pointnet.py +++ b/vis4d/op/base/pointnet.py @@ -311,7 +311,7 @@ def forward(self, features: torch.Tensor) -> PointNetEncoderOut: return PointNetEncoderOut( features=features, transformations=transforms, - pointwise_features=pointwise_features, + pointwise_features=pointwise_features, # pylint: disable=possibly-used-before-assignment, line-too-long ) diff --git a/vis4d/op/detect3d/bevformer/spatial_cross_attention.py b/vis4d/op/detect3d/bevformer/spatial_cross_attention.py index 3eb7fde02..1840d797a 100644 --- a/vis4d/op/detect3d/bevformer/spatial_cross_attention.py +++ b/vis4d/op/detect3d/bevformer/spatial_cross_attention.py @@ -59,7 +59,6 @@ def forward( spatial_shapes: Tensor, level_start_index: Tensor, bev_mask: Tensor, - residual: Tensor | None = None, query_pos: Tensor | None = None, ) -> Tensor: """Forward Function of Detr3DCrossAtten. @@ -82,17 +81,14 @@ def forward( as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. bev_mask (Tensor): The mask of BEV features with shape (num_query, bs, num_levels, h, w). - residual (Tensor): The tensor used for addition, with the - same shape as `x`. Default None. If None, `x` will be used. query_pos (Tensor): The positional encoding for `query`. Default None. Returns: Tensor: Forwarded results with shape [num_query, bs, embed_dims]. """ - if residual is None: - inp_residual = query - slots = torch.zeros_like(query) + inp_residual = query + slots = torch.zeros_like(query) if query_pos is not None: query = query + query_pos diff --git a/vis4d/op/detect3d/util.py b/vis4d/op/detect3d/util.py index d24e49d3a..80c646200 100644 --- a/vis4d/op/detect3d/util.py +++ b/vis4d/op/detect3d/util.py @@ -111,5 +111,7 @@ def batched_nms_rotated( raise RuntimeError( "Please install vis4d_cuda_ops to use batched_nms_rotated" ) - keep = nms_rotated(boxes_for_nms, scores, iou_threshold) + keep = nms_rotated( # pylint: disable=possibly-used-before-assignment + boxes_for_nms, scores, iou_threshold + ) return keep diff --git a/vis4d/op/geometry/rotation.py b/vis4d/op/geometry/rotation.py index 075545147..b605ecc56 100644 --- a/vis4d/op/geometry/rotation.py +++ b/vis4d/op/geometry/rotation.py @@ -139,6 +139,7 @@ def _axis_angle_rotation(axis: str, angle: Tensor) -> Tensor: Returns: Rotation matrices as tensor of shape (..., 3, 3). """ + assert axis in {"X", "Y", "Z"}, f"Invalid axis {axis}." cos = torch.cos(angle) sin = torch.sin(angle) one = torch.ones_like(angle) @@ -146,9 +147,9 @@ def _axis_angle_rotation(axis: str, angle: Tensor) -> Tensor: if axis == "X": rot_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos) - if axis == "Y": + elif axis == "Y": rot_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos) - if axis == "Z": + else: rot_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one) return torch.stack(rot_flat, -1).reshape(angle.shape + (3, 3)) diff --git a/vis4d/op/layer/ms_deform_attn.py b/vis4d/op/layer/ms_deform_attn.py index 1de35d629..9acb05566 100644 --- a/vis4d/op/layer/ms_deform_attn.py +++ b/vis4d/op/layer/ms_deform_attn.py @@ -19,6 +19,8 @@ if VIS4D_CUDA_OPS_AVAILABLE: from vis4d_cuda_ops import ms_deform_attn_backward, ms_deform_attn_forward +else: + raise ImportError("vis4d_cuda_ops is not installed.") class MSDeformAttentionFunction(Function): # pragma: no cover diff --git a/vis4d/op/mask/util.py b/vis4d/op/mask/util.py index 5723cacb4..02a768b78 100644 --- a/vis4d/op/mask/util.py +++ b/vis4d/op/mask/util.py @@ -80,8 +80,11 @@ def _do_paste_mask( img_masks = F.grid_sample(masks, grid, align_corners=False) if skip_empty: - return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) - return img_masks[:, 0], () + return img_masks[:, 0], ( # pylint: disable=unsubscriptable-object + slice(y0_int, y1_int), + slice(x0_int, x1_int), + ) + return img_masks[:, 0], () # pylint: disable=unsubscriptable-object def paste_masks_in_image( diff --git a/vis4d/pl/run.py b/vis4d/pl/run.py index c24924dab..e2f3f1b36 100644 --- a/vis4d/pl/run.py +++ b/vis4d/pl/run.py @@ -6,7 +6,7 @@ import os.path as osp import torch -from absl import app +from absl import app # pylint: disable=no-name-in-module from lightning.fabric.utilities.exceptions import MisconfigurationException from lightning.pytorch import Callback from torch.utils.collect_env import get_pretty_env_info diff --git a/vis4d/pl/training_module.py b/vis4d/pl/training_module.py index 218921639..8cc781535 100644 --- a/vis4d/pl/training_module.py +++ b/vis4d/pl/training_module.py @@ -155,7 +155,14 @@ def test_step( # pylint: disable=arguments-differ,line-too-long,unused-argument self.model, self.test_data_connector ) - flop_analyzer = FlopCountAnalysis(flops_model, flatten_inputs) + if not FVCORE_AVAILABLE: + raise RuntimeError( + "Please install fvcore to compute FLOPs of the model." + ) + + flop_analyzer = FlopCountAnalysis( # pylint: disable=possibly-used-before-assignment, line-too-long + flops_model, flatten_inputs + ) flop_analyzer.set_op_handle(**{k: None for k in IGNORED_OPS}) diff --git a/vis4d/zoo/base/models/qdtrack.py b/vis4d/zoo/base/models/qdtrack.py index ef7eca961..89b2e5848 100644 --- a/vis4d/zoo/base/models/qdtrack.py +++ b/vis4d/zoo/base/models/qdtrack.py @@ -173,6 +173,8 @@ def get_qdtrack_yolox_cfg( in_dim = 256 elif model_type == "xlarge": in_dim = 320 + else: + raise ValueError(f"Invalid model type: {model_type}") model = class_config( YOLOXQDTrack, num_classes=num_classes, diff --git a/vis4d/zoo/base/models/yolox.py b/vis4d/zoo/base/models/yolox.py index 91a18b513..044650deb 100644 --- a/vis4d/zoo/base/models/yolox.py +++ b/vis4d/zoo/base/models/yolox.py @@ -137,6 +137,8 @@ def get_model_setting(model_type: str) -> tuple[float, float, int, list[int]]: elif model_type == "xlarge": deepen_factor, widen_factor, num_csp_blocks = 1.33, 1.25, 4 in_channels = [320, 640, 1280] + else: + raise ValueError(f"Unknown model type: {model_type}") return deepen_factor, widen_factor, num_csp_blocks, in_channels diff --git a/vis4d/zoo/bevformer/data.py b/vis4d/zoo/bevformer/data.py index d72f74bd1..1bf164272 100644 --- a/vis4d/zoo/bevformer/data.py +++ b/vis4d/zoo/bevformer/data.py @@ -159,6 +159,8 @@ def get_nusc_cfg( mean = [103.530, 116.280, 123.675] std = [1.0, 1.0, 1.0] image_channel_mode = "BGR" + else: + raise ValueError(f"Unknown style {style}") if version == "v1.0-mini": # pragma: no cover assert train_split == "mini_train" diff --git a/vis4d/zoo/run.py b/vis4d/zoo/run.py index 7cd421af0..6251740a0 100644 --- a/vis4d/zoo/run.py +++ b/vis4d/zoo/run.py @@ -2,7 +2,7 @@ from __future__ import annotations -from absl import app +from absl import app # pylint: disable=no-name-in-module from vis4d.common import ArgsType from vis4d.zoo import AVAILABLE_MODELS From e3aa2b28118709f0ccd5bf7b01ea8b905eba11a2 Mon Sep 17 00:00:00 2001 From: RoyYang0714 Date: Wed, 15 May 2024 16:22:37 +0200 Subject: [PATCH 12/12] feat: Update version. --- pyproject.toml | 4 ++-- vis4d/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8cc162d41..9a06910a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,11 +112,11 @@ plugins = ["numpy.typing.mypy_plugin"] [project] name = "vis4d" -version = "0.1.1" +version = "0.1.2" authors = [{name = "VIS @ ETH", email = "i@yf.io"}] description = "Vis4D Python package for Visual 4D scene understanding" readme = "README.md" -license = { text="Apache 2.0"} +license = {text = "Apache 2.0"} requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", diff --git a/vis4d/__init__.py b/vis4d/__init__.py index c407efa37..279ebc78c 100644 --- a/vis4d/__init__.py +++ b/vis4d/__init__.py @@ -7,7 +7,7 @@ import logging -__version__ = "0.1.1" +__version__ = "0.1.2" _root_logger = logging.getLogger() _logger = logging.getLogger(__name__)