From f5f122425e4e2282b6d9d79a9ec11a627a96f606 Mon Sep 17 00:00:00 2001 From: Luca Soldaini Date: Mon, 30 Dec 2024 20:25:37 +0000 Subject: [PATCH] small tweaks --- configs/cc-news/mix-deupe-by-year.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/cc-news/mix-deupe-by-year.yaml b/configs/cc-news/mix-deupe-by-year.yaml index 85856a82..65d07669 100644 --- a/configs/cc-news/mix-deupe-by-year.yaml +++ b/configs/cc-news/mix-deupe-by-year.yaml @@ -5,8 +5,8 @@ streams: attributes: &attributes - dedupe_ngrams_20_1 output: &output - max_size_in_bytes: 2_500_000_000 - path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse-year-dedupe/documents + max_size_in_bytes: 3_814_697_265 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-year_dedup/documents filter: &filter include: - >-