data_args.py

from typing import Optional
from dataclasses import dataclass, field


@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.
    """

    dataset_name: Optional[str] = field(default="squad_kor_v1", metadata={"help": "The name of the dataset to use."})

    sub_datasets: Optional[str] = field(default="", metadata={"help": "The names of the sub datasets to use"})
    sub_datasets_ratio: Optional[str] = field(default="", metadata={"help": "The ratios of the sub datasets"})

    overwrite_cache: bool = field(default=False, metadata={"help": "Overwrite the cached training and evaluation sets"})
    cache_file_name: Optional[str] = field(
        default=None,
        metadata={
            "help": "Specific cache file that using in preprocessing."
            "If not specified, cache file can be autogenerated everytime you run dataset.map()"
        },
    )

    preprocessing_num_workers: Optional[int] = field(
        default=None, metadata={"help": "The number of processes to use for the preprocessing."}
    )

    max_seq_length: int = field(
        default=384,
        metadata={
            "help": "The maximum total input sequence length after tokenization. Sequences longer "
            "than this will be truncated, sequences shorter will be padded."
        },
    )

    pad_to_max_length: bool = field(
        default=False,
        metadata={
            "help": "Whether to pad all samples to `max_seq_length`. "
            "If False, will pad the samples dynamically when batching to the maximum length in the batch (which can "
            "be faster on GPU but will be slower on TPU)."
        },
    )

    batch_size: int = field(default=32)

    doc_stride: int = field(
        default=128,
        metadata={"help": "When splitting up a long document into chunks, how much stride to take between chunks."},
    )

    max_answer_length: int = field(
        default=30,
        metadata={
            "help": "The maximum length of an answer that can be generated. This is needed because the start "
            "and end predictions are not conditioned on one another."
        },
    )