input_field

Signed-off-by: Sarah Yurick <[email protected]>

config/fasttext_langid.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   - name: nemo_curator.filters.classifier_filter.FastTextLangId
     log_score: True

config/fasttext_quality_filter.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   - name: nemo_curator.filters.classifier_filter.FastTextQualityFilter
     params:

config/heuristic_filter_code.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   # The filters below define a chain of heuristic filters to be applied to each document in a corpus.
   # This particular cascade of filters is intended to filter Python code data.

config/heuristic_filter_en.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   # The filters below define a chain of heuristic filters to be applied to each document in a corpus.
   # This particular cascade of filters is intended to filter English language data.

config/heuristic_filter_non-en.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   # The filters below define a chain of heuristic filters to be applied to each document in a corpus.
   # This particular cascade of filters is intended to filter generic non-English data that use spaces for separating words.

nemo_curator/modules/fuzzy_dedup.py

@@ -786,11 +786,11 @@ def _combine_multiple_ids(
         output_df = input_df.copy()[input_df.columns.difference(id_fields)]
 
         output_df[output_id_field] = input_df[id_fields[0]].astype(str)
-        for input_field in id_fields[1:]:
+        for id_field in id_fields[1:]:
             output_df[output_id_field] = output_df[output_id_field] = (
                 input_df[id_fields[0]].astype(str)
                 + "-"
-                + input_df[input_field].astype(str)
+                + input_df[id_field].astype(str)
             )
 
         return output_df

nemo_curator/utils/config_utils.py

@@ -45,7 +45,7 @@ def build_filter(filter_config):
             doc_filter._name if filter_config.get("log_score", False) else None
         )
         filter_stage = nemo_curator.ScoreFilter(
-            doc_filter, filter_config.get("input_field"), score_field=score_field
+            doc_filter, filter_config.get("text_field"), score_field=score_field
         )
 
     return filter_stage
@@ -57,13 +57,13 @@ def build_filter_pipeline(filter_config_file):
         filter_params = yaml.load(config_file, Loader=yaml.FullLoader)
 
     filters = []
-    text_field = filter_params.get("input_field")
+    text_field = filter_params.get("text_field")
     for nc_filter_config in filter_params.get("filters"):
         if (
-            "input_field" not in nc_filter_config
-            or nc_filter_config["input_field"] is None
+            "text_field" not in nc_filter_config
+            or nc_filter_config["text_field"] is None
         ):
-            nc_filter_config["input_field"] = text_field
+            nc_filter_config["text_field"] = text_field
         new_filter = build_filter(nc_filter_config)
         filters.append(new_filter)
 

tutorials/pretraining-data-curation/config/heuristic_filter_en.yaml

@@ -1,4 +1,4 @@
-input_field: raw_content
+text_field: raw_content
 filters:
   - name: nemo_curator.filters.heuristic_filter.NonAlphaNumericFilter
     params:

tutorials/single_node_tutorial/config/heuristic_filter_non-en.yaml

@@ -1,4 +1,4 @@
-input_field: text
+text_field: text
 filters:
   # The filters below define a chain of heuristic filters to be applied to each document in a corpus.
   # This particular cascade of filters is intended to filter generic non-English data that use spaces for separating words.