redpajama-arxiv.yaml

# Process config example for arXiv dataset

# global parameters
project_name: 'arXiv'
dataset_path: '/path/to/your/dataset/dir/or/file'  # path to your dataset directory or file
np: 32  # number of subprocess to process your dataset

export_path: '/path/to/your/exported/dataset/file'

# process schedule
# a list of several process operators with their arguments
process:
  - remove_header_mapper:
      drop_no_head: true
  - remove_comments_mapper:
      doc_type: ['md', 'tex']
      inline: true
      multiline: true
  - remove_bibliography_mapper:
  - expand_macro_mapper:
  - text_length_filter:
      min_len: 1