Update LSF Finetune

README.md

@@ -162,14 +162,14 @@ Firstly, let's see how to use Uni2TS to fine-tune a pre-trained model on your cu
 Uni2TS uses the [Hugging Face datasets library](https://github.com/huggingface/datasets) to handle data loading, and we first need to convert your dataset into the Uni2TS format. 
 If your dataset is a simple pandas DataFrame, we can easily process your dataset with the following script.
 We'll use the ETTh1 dataset from the popular [Long Sequence Forecasting benchmark](https://github.com/thuml/Time-Series-Library) for this example.
-For more complex use cases, see [this notebook](example/prepare_data.ipynb) for more in-depth examples on how to use your custom dataset with Uni2TS.
+For more complex use cases, see [this notebook](example/prepare_data.ipynb) for more in-depth examples on how to use your custom dataset with Uni2TS. For formal LSF finetuning experiments based on original configurations, see [this folder](./project/moirai-1) for shell scripts and more detailed examples.
 
 1. To begin the process, add the path to the directory where you want to save the processed dataset into the ```.env``` file.
 ```shell
 echo "CUSTOM_DATA_PATH=PATH_TO_SAVE" >> .env
 ```
 
-2. Run the following script to process the dataset into the required format. For the ```dataset_type``` option, we support `wide`, `long` and `wide_multivariate`.
+2. Run the following script to process the dataset into the required format. For the ```dataset_type``` option, we support `wide`, `long` and `wide_multivariate`. 
 ```shell
 python -m uni2ts.data.builder.simple ETTh1 dataset/ETT-small/ETTh1.csv --dataset_type wide
 ```
@@ -181,14 +181,33 @@ The validation set will be saved as DATASET_NAME_eval.
 python -m uni2ts.data.builder.simple ETTh1 dataset/ETT-small/ETTh1.csv --date_offset '2017-10-23 23:00:00'
 ```
 
-3. Finally, we can simply run the fine-tuning script with the appropriate [training](cli/conf/finetune/data/etth1.yaml) and [validation](cli/conf/finetune/val_data/etth1.yaml) data configuration files.
+In some cases, we may want to normalize the data using the mean and std computed from the training dataset. This can be achieved by setting the ```--normalize``` argument.
+
+```shell
+python -m uni2ts.data.builder.simple ETTh1 dataset/ETT-small/ETTh1.csv --date_offset '2017-10-23 23:00:00' --normalize
+```
+
+
+3. Finally, we can simply run the fine-tuning script with the appropriate [training](cli/conf/finetune/data/etth1.yaml) and [validation](cli/conf/finetune/val_data/etth1.yaml) data configuration files. Forecasting configurations such as patch size, context length and prediction length need to be specified by users.  Since ```dataset_type``` is ```wide```, ```data.mode``` is set to `S` for univariate setup.
 ```shell
 python -m cli.train \
   -cp conf/finetune \
-  run_name=example_run \ 
-  model=moirai_1.0_R_small \ 
-  data=etth1 \ 
-  val_data=etth1  
+  exp_name=example_lsf \
+  run_name=example_run \
+  model=moirai_1.0_R_small \
+  model.patch_size=32 \
+  model.context_length=1000 \
+  model.prediction_length=96 \
+  data=etth1 \
+  data.patch_size=32 \
+  data.context_length=1000 \
+  data.prediction_length=96 \
+  data.mode=S \
+  val_data=etth1 \
+  val_data.patch_size=32 \
+  val_data.context_length=1000 \
+  val_data.prediction_length=96 \
+  val_data.mode=S
 ```
 
 ### Evaluation

cli/conf/finetune/data/etth1.yaml

@@ -1,3 +1,8 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: ETTh1
-weight: 1000
+train_length: 8640
+prediction_length: ???
+context_length: ???
+patch_size: ???
+mode: ???
+distance: 1

cli/conf/finetune/data/etth2.yaml

@@ -0,0 +1,8 @@
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
+dataset: ETTh2
+train_length: 8640
+prediction_length: ???
+context_length: ???
+patch_size: ???
+mode: ???
+distance: 1

cli/conf/finetune/data/ettm1.yaml

@@ -0,0 +1,8 @@
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
+dataset: ETTm1
+train_length: 34560
+prediction_length: ???
+context_length: ???
+patch_size: ???
+mode: ???
+distance: 1

cli/conf/finetune/data/ettm2.yaml

@@ -0,0 +1,8 @@
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
+dataset: ETTm2
+train_length: 34560
+prediction_length: ???
+context_length: ???
+patch_size: ???
+mode: ???
+distance: 1

cli/conf/finetune/default.yaml

@@ -1,11 +1,12 @@
 hydra:
   run:
-    dir: outputs/finetune/${hydra:runtime.choices.model}/${hydra:runtime.choices.data}/${run_name}
+    dir: outputs/finetune/${exp_name}/${hydra:runtime.choices.model}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
 defaults:
   - model: ???
   - data: ???
   - val_data: null
   - _self_
+exp_name: ???
 run_name: ???
 seed: 0
 tf32: true
@@ -32,31 +33,27 @@ trainer:
       mode: min
       save_top_k: 1
       every_n_epochs: 1
+      save_last: false
     - _target_: lightning.pytorch.callbacks.EarlyStopping
       monitor: val/PackedNLLLoss
       min_delta: 0.0
       patience: 3
       mode: min
       strict: false
       verbose: true
-  max_epochs: 100
+  max_epochs: 200
   enable_progress_bar: true
   accumulate_grad_batches: 1
   gradient_clip_val: 1.0
   gradient_clip_algorithm: norm
 train_dataloader:
   _target_: uni2ts.data.loader.DataLoader
-  batch_size: 128
+  batch_size: 512  # Since sequence packing is not used, one can use a large batch size when mode='S'
   batch_size_factor: 2.0
-  cycle: true
-  num_batches_per_epoch: 100
+  cycle: false  # Set it as false to loop over all batches per epoch
+  num_batches_per_epoch: null
   shuffle: true
   num_workers: 11
-  collate_fn:
-    _target_: uni2ts.data.loader.PackCollate
-    max_length: ${model.module_kwargs.max_seq_len}
-    seq_fields: ${cls_getattr:${model._target_},seq_fields}
-    pad_func_map: ${cls_getattr:${model._target_},pad_func_map}
   pin_memory: true
   drop_last: false
   fill_last: false
@@ -71,14 +68,9 @@ val_dataloader:
   num_batches_per_epoch: null
   shuffle: false
   num_workers: 11
-  collate_fn:
-    _target_: uni2ts.data.loader.PackCollate
-    max_length: ${model.module_kwargs.max_seq_len}
-    seq_fields: ${cls_getattr:${model._target_},seq_fields}
-    pad_func_map: ${cls_getattr:${model._target_},pad_func_map}
   pin_memory: false
   drop_last: false
-  fill_last: true
+  fill_last: false
   worker_init_fn: null
   prefetch_factor: 2
   persistent_workers: true

cli/conf/finetune/model/moirai_1.0_R_base.yaml

@@ -25,9 +25,17 @@ max_mask_ratio: 0.5
 max_dim: 128
 loss_func:
   _target_: uni2ts.loss.packed.PackedNLLLoss
-lr: 1e-3
+val_metric:
+  - _target_: uni2ts.loss.packed.PackedMSELoss
+  - _target_: uni2ts.loss.packed.PackedNRMSELoss
+    normalize: absolute_target_squared
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full

cli/conf/finetune/model/moirai_1.0_R_large.yaml

@@ -25,9 +25,13 @@ max_mask_ratio: 0.5
 max_dim: 128
 loss_func:
   _target_: uni2ts.loss.packed.PackedNLLLoss
-lr: 1e-3
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full

cli/conf/finetune/model/moirai_1.0_R_small.yaml

@@ -29,9 +29,13 @@ val_metric:
   - _target_: uni2ts.loss.packed.PackedMSELoss
   - _target_: uni2ts.loss.packed.PackedNRMSELoss
     normalize: absolute_target_squared
-lr: 1e-3
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full

cli/conf/finetune/model/moirai_1.1_R_base.yaml

@@ -25,9 +25,17 @@ max_mask_ratio: 0.5
 max_dim: 128
 loss_func:
   _target_: uni2ts.loss.packed.PackedNLLLoss
-lr: 1e-3
+val_metric:
+  - _target_: uni2ts.loss.packed.PackedMSELoss
+  - _target_: uni2ts.loss.packed.PackedNRMSELoss
+    normalize: absolute_target_squared
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full

cli/conf/finetune/model/moirai_1.1_R_large.yaml

@@ -25,9 +25,13 @@ max_mask_ratio: 0.5
 max_dim: 128
 loss_func:
   _target_: uni2ts.loss.packed.PackedNLLLoss
-lr: 1e-3
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full

cli/conf/finetune/model/moirai_1.1_R_small.yaml

@@ -29,9 +29,13 @@ val_metric:
   - _target_: uni2ts.loss.packed.PackedMSELoss
   - _target_: uni2ts.loss.packed.PackedNRMSELoss
     normalize: absolute_target_squared
-lr: 1e-3
+lr: 5e-7
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
-num_warmup_steps: 0
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full