From 80b2587d0c078b67045af89f5406f7649740e57e Mon Sep 17 00:00:00 2001 From: Judyxujj Date: Mon, 10 Jun 2024 17:51:26 +0200 Subject: [PATCH] update --- 2024-dynamic-encoder-size/README.md | 4 +- .../1_supernet_1_subnet/returnn.config | 425 +++++++++++++++ .../1_supernet_2_subnets/returnn.config | 425 +++++++++++++++ .../1_supernet_3_subnets/returnn.config | 499 ++++++++++++++++++ .../returnn.config | 0 5 files changed, 1352 insertions(+), 1 deletion(-) create mode 100644 2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_1_subnet/returnn.config create mode 100644 2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_2_subnets/returnn.config create mode 100644 2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_3_subnets/returnn.config rename 2024-dynamic-encoder-size/TED-LIUM-v2/simple_top_k/{1_supernet_1_subnets => 1_supernet_1_subnet}/returnn.config (100%) diff --git a/2024-dynamic-encoder-size/README.md b/2024-dynamic-encoder-size/README.md index d1326a17..28f6fd28 100644 --- a/2024-dynamic-encoder-size/README.md +++ b/2024-dynamic-encoder-size/README.md @@ -12,4 +12,6 @@ We use models parts from [i6-models](https://github.com/rwth-i6/i6_models/tree/j ConformerCTCModel, ConformerCTCConfig and train_step in returnn config is defined in [here](https://github.com/rwth-i6/i6_experiments/blob/main/users/jxu/experiments/ctc/tedlium2/pytorch_networks/dynamic_encoder_size/simple_topk_refactored/jointly_train_simple_top_k_layerwise.py) -### TED-LIUM-v2 Iterative-Zero-Out \ No newline at end of file +### TED-LIUM-v2 Iterative-Zero-Out + +ConformerCTCModel, ConformerCTCConfig and train_step in returnn config is defined in [here](https://github.com/rwth-i6/i6_experiments/blob/main/users/jxu/experiments/ctc/tedlium2/pytorch_networks/dynamic_encoder_size/iterative_zero_out_refactored/jointly_train_iterative_zero_out_layerwise.py) diff --git a/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_1_subnet/returnn.config b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_1_subnet/returnn.config new file mode 100644 index 00000000..fb92d59b --- /dev/null +++ b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_1_subnet/returnn.config @@ -0,0 +1,425 @@ +#!rnn.py + + +import numpy as np + +backend = "torch" +batch_size = 15000 +batching = "random" +cache_size = "0" +cleanup_old_models = True +dev = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.U8uxDBlFQCdV/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.H5yw1ICx8RsS/output/targets.hdf" + ], + "partition_epoch": 1, + "seq_ordering": "sorted", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +device = "gpu" +extern_data = {"data": {"dim": 50}, "targets": {"dim": 79, "sparse": True}} +gradient_clip = 0.0 +gradient_noise = 0.0 +learning_rate_file = "learning_rates" +learning_rates = [ + 4e-06, + 7.6e-06, + 1.1200000000000001e-05, + 1.48e-05, + 1.84e-05, + 2.2e-05, + 2.5600000000000002e-05, + 2.92e-05, + 3.2800000000000004e-05, + 3.6400000000000004e-05, + 4e-05, + 4.36e-05, + 4.720000000000001e-05, + 5.080000000000001e-05, + 5.440000000000001e-05, + 5.800000000000001e-05, + 6.16e-05, + 6.520000000000001e-05, + 6.88e-05, + 7.240000000000001e-05, + 7.6e-05, + 7.960000000000001e-05, + 8.32e-05, + 8.680000000000001e-05, + 9.040000000000002e-05, + 9.400000000000001e-05, + 9.760000000000001e-05, + 0.00010120000000000001, + 0.00010480000000000001, + 0.0001084, + 0.00011200000000000001, + 0.0001156, + 0.00011920000000000001, + 0.0001228, + 0.0001264, + 0.00013, + 0.0001336, + 0.0001372, + 0.0001408, + 0.00014439999999999999, + 0.000148, + 0.0001516, + 0.0001552, + 0.0001588, + 0.0001624, + 0.000166, + 0.0001696, + 0.0001732, + 0.00017680000000000001, + 0.0001804, + 0.000184, + 0.0001876, + 0.0001912, + 0.0001948, + 0.0001984, + 0.000202, + 0.0002056, + 0.00020920000000000002, + 0.0002128, + 0.0002164, + 0.00022, + 0.00022360000000000001, + 0.0002272, + 0.0002308, + 0.0002344, + 0.000238, + 0.00024160000000000002, + 0.0002452, + 0.00024880000000000003, + 0.0002524, + 0.000256, + 0.0002596, + 0.0002632, + 0.00026680000000000003, + 0.0002704, + 0.000274, + 0.0002776, + 0.0002812, + 0.0002848, + 0.0002884, + 0.000292, + 0.00029560000000000003, + 0.0002992, + 0.0003028, + 0.0003064, + 0.00031, + 0.00031360000000000003, + 0.0003172, + 0.0003208, + 0.0003244, + 0.000328, + 0.00033160000000000004, + 0.0003352, + 0.0003388, + 0.00034240000000000003, + 0.000346, + 0.00034960000000000004, + 0.0003532, + 0.0003568, + 0.00036040000000000003, + 0.000364, + 0.0003676, + 0.0003712, + 0.0003748, + 0.00037840000000000004, + 0.000382, + 0.0003856, + 0.00038920000000000003, + 0.0003928, + 0.00039640000000000004, + 0.0004, + 0.00039640000000000004, + 0.0003928, + 0.00038920000000000003, + 0.0003856, + 0.000382, + 0.00037840000000000004, + 0.0003748, + 0.0003712, + 0.00036760000000000004, + 0.000364, + 0.00036040000000000003, + 0.0003568, + 0.0003532, + 0.00034960000000000004, + 0.000346, + 0.00034240000000000003, + 0.0003388, + 0.0003352, + 0.00033160000000000004, + 0.000328, + 0.0003244, + 0.0003208, + 0.0003172, + 0.00031360000000000003, + 0.00031, + 0.0003064, + 0.0003028, + 0.0002992, + 0.00029560000000000003, + 0.000292, + 0.0002884, + 0.0002848, + 0.0002812, + 0.0002776, + 0.00027400000000000005, + 0.0002704, + 0.0002668, + 0.0002632, + 0.0002596, + 0.00025600000000000004, + 0.0002524, + 0.0002488, + 0.0002452, + 0.00024160000000000002, + 0.000238, + 0.0002344, + 0.0002308, + 0.0002272, + 0.00022360000000000001, + 0.00022, + 0.0002164, + 0.0002128, + 0.00020920000000000002, + 0.0002056, + 0.000202, + 0.0001984, + 0.0001948, + 0.0001912, + 0.0001876, + 0.000184, + 0.0001804, + 0.00017680000000000001, + 0.0001732, + 0.0001696, + 0.000166, + 0.0001624, + 0.0001588, + 0.00015519999999999998, + 0.0001516, + 0.00014800000000000002, + 0.00014439999999999999, + 0.0001408, + 0.00013719999999999997, + 0.0001336, + 0.00013000000000000002, + 0.00012639999999999998, + 0.0001228, + 0.00011920000000000002, + 0.00011559999999999999, + 0.00011200000000000001, + 0.00010839999999999998, + 0.0001048, + 0.00010120000000000002, + 9.759999999999999e-05, + 9.400000000000001e-05, + 9.039999999999997e-05, + 8.68e-05, + 8.320000000000002e-05, + 7.959999999999998e-05, + 7.6e-05, + 7.239999999999997e-05, + 6.879999999999999e-05, + 6.520000000000001e-05, + 6.159999999999998e-05, + 5.8e-05, + 5.439999999999997e-05, + 5.079999999999999e-05, + 4.720000000000001e-05, + 4.3599999999999976e-05, + 3.9999999999999996e-05, + 3.640000000000002e-05, + 3.2799999999999984e-05, + 2.9200000000000005e-05, + 2.5599999999999972e-05, + 2.1999999999999993e-05, + 1.8400000000000014e-05, + 1.479999999999998e-05, + 1.1200000000000001e-05, + 7.599999999999968e-06, + 4e-06, + 3.862413793103448e-06, + 3.7248275862068965e-06, + 3.587241379310345e-06, + 3.449655172413793e-06, + 3.3120689655172415e-06, + 3.1744827586206894e-06, + 3.0368965517241378e-06, + 2.899310344827586e-06, + 2.7617241379310344e-06, + 2.6241379310344828e-06, + 2.4865517241379307e-06, + 2.348965517241379e-06, + 2.2113793103448274e-06, + 2.0737931034482757e-06, + 1.936206896551724e-06, + 1.7986206896551724e-06, + 1.6610344827586207e-06, + 1.523448275862069e-06, + 1.3858620689655174e-06, + 1.2482758620689657e-06, + 1.1106896551724136e-06, + 9.73103448275862e-07, + 8.355172413793103e-07, + 6.979310344827586e-07, + 5.60344827586207e-07, + 4.227586206896553e-07, + 2.8517241379310365e-07, + 1.4758620689655157e-07, + 1e-08, +] +log = ["./returnn.log"] +log_batch_size = True +log_verbosity = 5 +max_seqs = 128 +model = "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/training/ReturnnTrainingJob.IOudLdcvvkUP/output/models/epoch" +num_epochs = 250 +num_inputs = 50 +num_outputs = {"targets": 79} +optimizer = {"class": "adamw", "epsilon": 1e-16, "weight_decay": 0.001} +save_interval = 1 +target = "targets" +task = "train" +tf_log_memory_usage = True +train = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.jxsVEaCgJLqB/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.cEWylWI7BnXk/output/targets.hdf" + ], + "partition_epoch": 5, + "seq_ordering": "laplace:.1000", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +update_on_device = True +window = 1 +config = {} + +locals().update(**config) + +import os +import sys + +sys.path.insert(0, "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/recipe") +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.iterative_zero_out_refactored.jointly_train_iterative_zero_out_layerwise import ( + ConformerCTCModel, +) +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.iterative_zero_out_refactored.jointly_train_iterative_zero_out_layerwise import ( + ConformerCTCConfig, +) +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerEncoderConfig, +) +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1Config +from torch.nn.modules.activation import ReLU +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1 +from i6_models.config import ModuleFactoryV1 +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerBlockConfig, +) +from i6_models.parts.conformer.feedforward import ( + ConformerPositionwiseFeedForwardV1Config, +) +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.mhsa import ConformerMHSAV1Config +from i6_models.parts.conformer.convolution import ConformerConvolutionV1Config +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.norm import LayerNormNC +from torch.nn.modules.activation import Sigmoid + +cfg = ConformerCTCConfig( + conformer_cfg=ConformerEncoderConfig( + num_layers=12, + frontend=ModuleFactoryV1( + module_class=VGG4LayerActFrontendV1, + cfg=VGG4LayerActFrontendV1Config( + in_features=50, + conv1_channels=32, + conv2_channels=64, + conv3_channels=64, + conv4_channels=32, + conv_kernel_size=(3, 3), + conv_padding=None, + pool1_kernel_size=(2, 1), + pool1_stride=(2, 1), + pool1_padding=None, + pool2_kernel_size=(2, 1), + pool2_stride=(2, 1), + pool2_padding=None, + activation=ReLU(), + out_features=384, + ), + ), + block_cfg=ConformerBlockConfig( + ff_cfg=ConformerPositionwiseFeedForwardV1Config( + input_dim=384, hidden_dim=1536, dropout=0.2, activation=SiLU() + ), + mhsa_cfg=ConformerMHSAV1Config( + input_dim=384, num_att_heads=6, att_weights_dropout=0.1, dropout=0.2 + ), + conv_cfg=ConformerConvolutionV1Config( + channels=384, + kernel_size=31, + dropout=0.2, + activation=SiLU(), + norm=LayerNormNC((384,), eps=1e-05, elementwise_affine=True), + ), + layer_dropout=0, + modules=["ff", "conv", "mhsa", "ff"], + scales=[0.5, 1.0, 1.0, 0.5], + ), + num_layers_set=[24, 48], + layer_dropout_kwargs={"layer_dropout_stage_1": 0, "layer_dropout_stage_2": 0.3}, + layer_gate_activation=Sigmoid(), + ), + target_size=79, + recog_num_layers=48, + zero_out_kwargs={ + "num_steps_per_iter": [63000, 21000, 21000, 21000, 21000, 21000, 21000, 21000], + "num_zeroout_elements_per_iter": [3, 6, 9, 12, 15, 18, 21, 24], + "zeroout_val": -5, + }, +) +model_kwargs = {"cfg": cfg} + + +def get_model(epoch, step, **kwargs): + return ConformerCTCModel(epoch=epoch, step=step, **model_kwargs, **kwargs) + + +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.iterative_zero_out_refactored.jointly_train_iterative_zero_out_layerwise import ( + train_step, +) diff --git a/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_2_subnets/returnn.config b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_2_subnets/returnn.config new file mode 100644 index 00000000..6c4941c3 --- /dev/null +++ b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_2_subnets/returnn.config @@ -0,0 +1,425 @@ +#!rnn.py + + +import numpy as np + +backend = "torch" +batch_size = 15000 +batching = "random" +cache_size = "0" +cleanup_old_models = True +dev = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.U8uxDBlFQCdV/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.H5yw1ICx8RsS/output/targets.hdf" + ], + "partition_epoch": 1, + "seq_ordering": "sorted", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +device = "gpu" +extern_data = {"data": {"dim": 50}, "targets": {"dim": 79, "sparse": True}} +gradient_clip = 0.0 +gradient_noise = 0.0 +learning_rate_file = "learning_rates" +learning_rates = [ + 4e-06, + 7.6e-06, + 1.1200000000000001e-05, + 1.48e-05, + 1.84e-05, + 2.2e-05, + 2.5600000000000002e-05, + 2.92e-05, + 3.2800000000000004e-05, + 3.6400000000000004e-05, + 4e-05, + 4.36e-05, + 4.720000000000001e-05, + 5.080000000000001e-05, + 5.440000000000001e-05, + 5.800000000000001e-05, + 6.16e-05, + 6.520000000000001e-05, + 6.88e-05, + 7.240000000000001e-05, + 7.6e-05, + 7.960000000000001e-05, + 8.32e-05, + 8.680000000000001e-05, + 9.040000000000002e-05, + 9.400000000000001e-05, + 9.760000000000001e-05, + 0.00010120000000000001, + 0.00010480000000000001, + 0.0001084, + 0.00011200000000000001, + 0.0001156, + 0.00011920000000000001, + 0.0001228, + 0.0001264, + 0.00013, + 0.0001336, + 0.0001372, + 0.0001408, + 0.00014439999999999999, + 0.000148, + 0.0001516, + 0.0001552, + 0.0001588, + 0.0001624, + 0.000166, + 0.0001696, + 0.0001732, + 0.00017680000000000001, + 0.0001804, + 0.000184, + 0.0001876, + 0.0001912, + 0.0001948, + 0.0001984, + 0.000202, + 0.0002056, + 0.00020920000000000002, + 0.0002128, + 0.0002164, + 0.00022, + 0.00022360000000000001, + 0.0002272, + 0.0002308, + 0.0002344, + 0.000238, + 0.00024160000000000002, + 0.0002452, + 0.00024880000000000003, + 0.0002524, + 0.000256, + 0.0002596, + 0.0002632, + 0.00026680000000000003, + 0.0002704, + 0.000274, + 0.0002776, + 0.0002812, + 0.0002848, + 0.0002884, + 0.000292, + 0.00029560000000000003, + 0.0002992, + 0.0003028, + 0.0003064, + 0.00031, + 0.00031360000000000003, + 0.0003172, + 0.0003208, + 0.0003244, + 0.000328, + 0.00033160000000000004, + 0.0003352, + 0.0003388, + 0.00034240000000000003, + 0.000346, + 0.00034960000000000004, + 0.0003532, + 0.0003568, + 0.00036040000000000003, + 0.000364, + 0.0003676, + 0.0003712, + 0.0003748, + 0.00037840000000000004, + 0.000382, + 0.0003856, + 0.00038920000000000003, + 0.0003928, + 0.00039640000000000004, + 0.0004, + 0.00039640000000000004, + 0.0003928, + 0.00038920000000000003, + 0.0003856, + 0.000382, + 0.00037840000000000004, + 0.0003748, + 0.0003712, + 0.00036760000000000004, + 0.000364, + 0.00036040000000000003, + 0.0003568, + 0.0003532, + 0.00034960000000000004, + 0.000346, + 0.00034240000000000003, + 0.0003388, + 0.0003352, + 0.00033160000000000004, + 0.000328, + 0.0003244, + 0.0003208, + 0.0003172, + 0.00031360000000000003, + 0.00031, + 0.0003064, + 0.0003028, + 0.0002992, + 0.00029560000000000003, + 0.000292, + 0.0002884, + 0.0002848, + 0.0002812, + 0.0002776, + 0.00027400000000000005, + 0.0002704, + 0.0002668, + 0.0002632, + 0.0002596, + 0.00025600000000000004, + 0.0002524, + 0.0002488, + 0.0002452, + 0.00024160000000000002, + 0.000238, + 0.0002344, + 0.0002308, + 0.0002272, + 0.00022360000000000001, + 0.00022, + 0.0002164, + 0.0002128, + 0.00020920000000000002, + 0.0002056, + 0.000202, + 0.0001984, + 0.0001948, + 0.0001912, + 0.0001876, + 0.000184, + 0.0001804, + 0.00017680000000000001, + 0.0001732, + 0.0001696, + 0.000166, + 0.0001624, + 0.0001588, + 0.00015519999999999998, + 0.0001516, + 0.00014800000000000002, + 0.00014439999999999999, + 0.0001408, + 0.00013719999999999997, + 0.0001336, + 0.00013000000000000002, + 0.00012639999999999998, + 0.0001228, + 0.00011920000000000002, + 0.00011559999999999999, + 0.00011200000000000001, + 0.00010839999999999998, + 0.0001048, + 0.00010120000000000002, + 9.759999999999999e-05, + 9.400000000000001e-05, + 9.039999999999997e-05, + 8.68e-05, + 8.320000000000002e-05, + 7.959999999999998e-05, + 7.6e-05, + 7.239999999999997e-05, + 6.879999999999999e-05, + 6.520000000000001e-05, + 6.159999999999998e-05, + 5.8e-05, + 5.439999999999997e-05, + 5.079999999999999e-05, + 4.720000000000001e-05, + 4.3599999999999976e-05, + 3.9999999999999996e-05, + 3.640000000000002e-05, + 3.2799999999999984e-05, + 2.9200000000000005e-05, + 2.5599999999999972e-05, + 2.1999999999999993e-05, + 1.8400000000000014e-05, + 1.479999999999998e-05, + 1.1200000000000001e-05, + 7.599999999999968e-06, + 4e-06, + 3.862413793103448e-06, + 3.7248275862068965e-06, + 3.587241379310345e-06, + 3.449655172413793e-06, + 3.3120689655172415e-06, + 3.1744827586206894e-06, + 3.0368965517241378e-06, + 2.899310344827586e-06, + 2.7617241379310344e-06, + 2.6241379310344828e-06, + 2.4865517241379307e-06, + 2.348965517241379e-06, + 2.2113793103448274e-06, + 2.0737931034482757e-06, + 1.936206896551724e-06, + 1.7986206896551724e-06, + 1.6610344827586207e-06, + 1.523448275862069e-06, + 1.3858620689655174e-06, + 1.2482758620689657e-06, + 1.1106896551724136e-06, + 9.73103448275862e-07, + 8.355172413793103e-07, + 6.979310344827586e-07, + 5.60344827586207e-07, + 4.227586206896553e-07, + 2.8517241379310365e-07, + 1.4758620689655157e-07, + 1e-08, +] +log = ["./returnn.log"] +log_batch_size = True +log_verbosity = 5 +max_seqs = 128 +model = "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/training/ReturnnTrainingJob.TryLPoyqOkV6/output/models/epoch" +num_epochs = 250 +num_inputs = 50 +num_outputs = {"targets": 79} +optimizer = {"class": "adamw", "epsilon": 1e-16, "weight_decay": 0.001} +save_interval = 1 +target = "targets" +task = "train" +tf_log_memory_usage = True +train = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.jxsVEaCgJLqB/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.cEWylWI7BnXk/output/targets.hdf" + ], + "partition_epoch": 5, + "seq_ordering": "laplace:.1000", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +update_on_device = True +window = 1 +config = {} + +locals().update(**config) + +import os +import sys + +sys.path.insert(0, "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/recipe") +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + ConformerCTCModel, +) +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + ConformerCTCConfig, +) +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerEncoderConfig, +) +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1Config +from torch.nn.modules.activation import ReLU +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1 +from i6_models.config import ModuleFactoryV1 +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerBlockConfig, +) +from i6_models.parts.conformer.feedforward import ( + ConformerPositionwiseFeedForwardV1Config, +) +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.mhsa import ConformerMHSAV1Config +from i6_models.parts.conformer.convolution import ConformerConvolutionV1Config +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.norm import LayerNormNC +from torch.nn.modules.activation import Sigmoid + +cfg = ConformerCTCConfig( + conformer_cfg=ConformerEncoderConfig( + num_layers=12, + frontend=ModuleFactoryV1( + module_class=VGG4LayerActFrontendV1, + cfg=VGG4LayerActFrontendV1Config( + in_features=50, + conv1_channels=32, + conv2_channels=64, + conv3_channels=64, + conv4_channels=32, + conv_kernel_size=(3, 3), + conv_padding=None, + pool1_kernel_size=(2, 1), + pool1_stride=(2, 1), + pool1_padding=None, + pool2_kernel_size=(2, 1), + pool2_stride=(2, 1), + pool2_padding=None, + activation=ReLU(), + out_features=384, + ), + ), + block_cfg=ConformerBlockConfig( + ff_cfg=ConformerPositionwiseFeedForwardV1Config( + input_dim=384, hidden_dim=1536, dropout=0.2, activation=SiLU() + ), + mhsa_cfg=ConformerMHSAV1Config( + input_dim=384, num_att_heads=6, att_weights_dropout=0.1, dropout=0.2 + ), + conv_cfg=ConformerConvolutionV1Config( + channels=384, + kernel_size=31, + dropout=0.2, + activation=SiLU(), + norm=LayerNormNC((384,), eps=1e-05, elementwise_affine=True), + ), + layer_dropout=0, + modules=["ff", "conv", "mhsa", "ff"], + scales=[0.5, 1.0, 1.0, 0.5], + ), + num_layers_set=[16, 32, 48], + layer_dropout_kwargs={"layer_dropout_stage_1": 0, "layer_dropout_stage_2": 0.3}, + layer_gate_activation=Sigmoid(), + ), + target_size=79, + recog_num_layers=48, + zero_out_kwargs={ + "num_steps_per_iter": [42000, 25200, 25200, 25200, 25200, 25200, 25200, 25200], + "num_zeroout_elements_per_iter": [4, 8, 12, 16, 20, 24, 28, 32], + "zeroout_val": -5, + }, +) +model_kwargs = {"cfg": cfg} + + +def get_model(epoch, step, **kwargs): + return ConformerCTCModel(epoch=epoch, step=step, **model_kwargs, **kwargs) + + +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + train_step, +) diff --git a/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_3_subnets/returnn.config b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_3_subnets/returnn.config new file mode 100644 index 00000000..cbcade00 --- /dev/null +++ b/2024-dynamic-encoder-size/TED-LIUM-v2/iterative_zero_out/1_supernet_3_subnets/returnn.config @@ -0,0 +1,499 @@ +#!rnn.py + + +import numpy as np + +backend = "torch" +batch_size = 15000 +batching = "random" +cache_size = "0" +cleanup_old_models = True +dev = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.U8uxDBlFQCdV/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.H5yw1ICx8RsS/output/targets.hdf" + ], + "partition_epoch": 1, + "seq_ordering": "sorted", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +device = "gpu" +extern_data = {"data": {"dim": 50}, "targets": {"dim": 79, "sparse": True}} +gradient_clip = 0.0 +gradient_noise = 0.0 +learning_rate_file = "learning_rates" +learning_rates = [ + 4e-06, + 7.6e-06, + 1.1200000000000001e-05, + 1.48e-05, + 1.84e-05, + 2.2e-05, + 2.5600000000000002e-05, + 2.92e-05, + 3.2800000000000004e-05, + 3.6400000000000004e-05, + 4e-05, + 4.36e-05, + 4.720000000000001e-05, + 5.080000000000001e-05, + 5.440000000000001e-05, + 5.800000000000001e-05, + 6.16e-05, + 6.520000000000001e-05, + 6.88e-05, + 7.240000000000001e-05, + 7.6e-05, + 7.960000000000001e-05, + 8.32e-05, + 8.680000000000001e-05, + 9.040000000000002e-05, + 9.400000000000001e-05, + 9.760000000000001e-05, + 0.00010120000000000001, + 0.00010480000000000001, + 0.0001084, + 0.00011200000000000001, + 0.0001156, + 0.00011920000000000001, + 0.0001228, + 0.0001264, + 0.00013, + 0.0001336, + 0.0001372, + 0.0001408, + 0.00014439999999999999, + 0.000148, + 0.0001516, + 0.0001552, + 0.0001588, + 0.0001624, + 0.000166, + 0.0001696, + 0.0001732, + 0.00017680000000000001, + 0.0001804, + 0.000184, + 0.0001876, + 0.0001912, + 0.0001948, + 0.0001984, + 0.000202, + 0.0002056, + 0.00020920000000000002, + 0.0002128, + 0.0002164, + 0.00022, + 0.00022360000000000001, + 0.0002272, + 0.0002308, + 0.0002344, + 0.000238, + 0.00024160000000000002, + 0.0002452, + 0.00024880000000000003, + 0.0002524, + 0.000256, + 0.0002596, + 0.0002632, + 0.00026680000000000003, + 0.0002704, + 0.000274, + 0.0002776, + 0.0002812, + 0.0002848, + 0.0002884, + 0.000292, + 0.00029560000000000003, + 0.0002992, + 0.0003028, + 0.0003064, + 0.00031, + 0.00031360000000000003, + 0.0003172, + 0.0003208, + 0.0003244, + 0.000328, + 0.00033160000000000004, + 0.0003352, + 0.0003388, + 0.00034240000000000003, + 0.000346, + 0.00034960000000000004, + 0.0003532, + 0.0003568, + 0.00036040000000000003, + 0.000364, + 0.0003676, + 0.0003712, + 0.0003748, + 0.00037840000000000004, + 0.000382, + 0.0003856, + 0.00038920000000000003, + 0.0003928, + 0.00039640000000000004, + 0.0004, + 0.00039640000000000004, + 0.0003928, + 0.00038920000000000003, + 0.0003856, + 0.000382, + 0.00037840000000000004, + 0.0003748, + 0.0003712, + 0.00036760000000000004, + 0.000364, + 0.00036040000000000003, + 0.0003568, + 0.0003532, + 0.00034960000000000004, + 0.000346, + 0.00034240000000000003, + 0.0003388, + 0.0003352, + 0.00033160000000000004, + 0.000328, + 0.0003244, + 0.0003208, + 0.0003172, + 0.00031360000000000003, + 0.00031, + 0.0003064, + 0.0003028, + 0.0002992, + 0.00029560000000000003, + 0.000292, + 0.0002884, + 0.0002848, + 0.0002812, + 0.0002776, + 0.00027400000000000005, + 0.0002704, + 0.0002668, + 0.0002632, + 0.0002596, + 0.00025600000000000004, + 0.0002524, + 0.0002488, + 0.0002452, + 0.00024160000000000002, + 0.000238, + 0.0002344, + 0.0002308, + 0.0002272, + 0.00022360000000000001, + 0.00022, + 0.0002164, + 0.0002128, + 0.00020920000000000002, + 0.0002056, + 0.000202, + 0.0001984, + 0.0001948, + 0.0001912, + 0.0001876, + 0.000184, + 0.0001804, + 0.00017680000000000001, + 0.0001732, + 0.0001696, + 0.000166, + 0.0001624, + 0.0001588, + 0.00015519999999999998, + 0.0001516, + 0.00014800000000000002, + 0.00014439999999999999, + 0.0001408, + 0.00013719999999999997, + 0.0001336, + 0.00013000000000000002, + 0.00012639999999999998, + 0.0001228, + 0.00011920000000000002, + 0.00011559999999999999, + 0.00011200000000000001, + 0.00010839999999999998, + 0.0001048, + 0.00010120000000000002, + 9.759999999999999e-05, + 9.400000000000001e-05, + 9.039999999999997e-05, + 8.68e-05, + 8.320000000000002e-05, + 7.959999999999998e-05, + 7.6e-05, + 7.239999999999997e-05, + 6.879999999999999e-05, + 6.520000000000001e-05, + 6.159999999999998e-05, + 5.8e-05, + 5.439999999999997e-05, + 5.079999999999999e-05, + 4.720000000000001e-05, + 4.3599999999999976e-05, + 3.9999999999999996e-05, + 3.640000000000002e-05, + 3.2799999999999984e-05, + 2.9200000000000005e-05, + 2.5599999999999972e-05, + 2.1999999999999993e-05, + 1.8400000000000014e-05, + 1.479999999999998e-05, + 1.1200000000000001e-05, + 7.599999999999968e-06, + 4e-06, + 3.862413793103448e-06, + 3.7248275862068965e-06, + 3.587241379310345e-06, + 3.449655172413793e-06, + 3.3120689655172415e-06, + 3.1744827586206894e-06, + 3.0368965517241378e-06, + 2.899310344827586e-06, + 2.7617241379310344e-06, + 2.6241379310344828e-06, + 2.4865517241379307e-06, + 2.348965517241379e-06, + 2.2113793103448274e-06, + 2.0737931034482757e-06, + 1.936206896551724e-06, + 1.7986206896551724e-06, + 1.6610344827586207e-06, + 1.523448275862069e-06, + 1.3858620689655174e-06, + 1.2482758620689657e-06, + 1.1106896551724136e-06, + 9.73103448275862e-07, + 8.355172413793103e-07, + 6.979310344827586e-07, + 5.60344827586207e-07, + 4.227586206896553e-07, + 2.8517241379310365e-07, + 1.4758620689655157e-07, + 1e-08, +] +log = ["./returnn.log"] +log_batch_size = True +log_verbosity = 5 +max_seqs = 128 +model = "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/training/ReturnnTrainingJob.aLCjBKpJEw1L/output/models/epoch" +num_epochs = 250 +num_inputs = 50 +num_outputs = {"targets": 79} +optimizer = {"class": "adamw", "epsilon": 1e-16, "weight_decay": 0.001} +save_interval = 1 +target = "targets" +task = "train" +tf_log_memory_usage = True +train = { + "class": "MetaDataset", + "data_map": {"data": ("features", "data"), "targets": ("targets", "data")}, + "datasets": { + "features": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_core/returnn/hdf/ReturnnDumpHDFJob.jxsVEaCgJLqB/output/data.hdf" + ], + "use_cache_manager": True, + }, + "targets": { + "class": "HDFDataset", + "files": [ + "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/work/i6_experiments/users/berger/recipe/returnn/hdf/BlissCorpusToTargetHdfJob.cEWylWI7BnXk/output/targets.hdf" + ], + "partition_epoch": 5, + "seq_ordering": "laplace:.1000", + "use_cache_manager": True, + }, + }, + "seq_order_control_dataset": "targets", +} +update_on_device = True +window = 1 +config = {} + +locals().update(**config) + +import os +import sys + +sys.path.insert(0, "/u/jxu/setups/tedlium2/2023-07-11--ctc-tedlium/recipe") +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + ConformerCTCModel, +) +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + ConformerCTCConfig, +) +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerEncoderConfig, +) +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1Config +from torch.nn.modules.activation import ReLU +from i6_models.parts.frontend.vgg_act import VGG4LayerActFrontendV1 +from i6_models.config import ModuleFactoryV1 +from i6_models.assemblies.conformer_with_dynamic_model_size.selection_with_iterative_zero_out import ( + ConformerBlockConfig, +) +from i6_models.parts.conformer.feedforward import ( + ConformerPositionwiseFeedForwardV1Config, +) +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.mhsa import ConformerMHSAV1Config +from i6_models.parts.conformer.convolution import ConformerConvolutionV1Config +from torch.nn.modules.activation import SiLU +from i6_models.parts.conformer.norm import LayerNormNC +from torch.nn.modules.activation import Sigmoid + +cfg = ConformerCTCConfig( + conformer_cfg=ConformerEncoderConfig( + num_layers=12, + frontend=ModuleFactoryV1( + module_class=VGG4LayerActFrontendV1, + cfg=VGG4LayerActFrontendV1Config( + in_features=50, + conv1_channels=32, + conv2_channels=64, + conv3_channels=64, + conv4_channels=32, + conv_kernel_size=(3, 3), + conv_padding=None, + pool1_kernel_size=(2, 1), + pool1_stride=(2, 1), + pool1_padding=None, + pool2_kernel_size=(2, 1), + pool2_stride=(2, 1), + pool2_padding=None, + activation=ReLU(), + out_features=384, + ), + ), + block_cfg=ConformerBlockConfig( + ff_cfg=ConformerPositionwiseFeedForwardV1Config( + input_dim=384, hidden_dim=1536, dropout=0.2, activation=SiLU() + ), + mhsa_cfg=ConformerMHSAV1Config( + input_dim=384, num_att_heads=6, att_weights_dropout=0.1, dropout=0.2 + ), + conv_cfg=ConformerConvolutionV1Config( + channels=384, + kernel_size=31, + dropout=0.2, + activation=SiLU(), + norm=LayerNormNC((384,), eps=1e-05, elementwise_affine=True), + ), + layer_dropout=0, + modules=["ff", "conv", "mhsa", "ff"], + scales=[0.5, 1.0, 1.0, 0.5], + ), + num_layers_set=[12, 24, 36, 48], + layer_dropout_kwargs={"layer_dropout_stage_1": 0, "layer_dropout_stage_2": 0.3}, + layer_gate_activation=Sigmoid(), + ), + target_size=79, + recog_num_layers=48, + zero_out_kwargs={ + "num_steps_per_iter": [ + 14000, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + 5600, + ], + "num_zeroout_elements_per_iter": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + ], + "zeroout_val": -5, + }, +) +model_kwargs = {"cfg": cfg} + + +def get_model(epoch, step, **kwargs): + return ConformerCTCModel(epoch=epoch, step=step, **model_kwargs, **kwargs) + + +from i6_experiments.users.jxu.experiments.ctc.tedlium2.pytorch_networks.dynamic_encoder_size.zeroout_with_new_i6_models.joint_train_two_model_iterative_zero_out_modwise import ( + train_step, +) diff --git a/2024-dynamic-encoder-size/TED-LIUM-v2/simple_top_k/1_supernet_1_subnets/returnn.config b/2024-dynamic-encoder-size/TED-LIUM-v2/simple_top_k/1_supernet_1_subnet/returnn.config similarity index 100% rename from 2024-dynamic-encoder-size/TED-LIUM-v2/simple_top_k/1_supernet_1_subnets/returnn.config rename to 2024-dynamic-encoder-size/TED-LIUM-v2/simple_top_k/1_supernet_1_subnet/returnn.config