From fa984bb212b1af43bd38b9492a63e49f124c1bca Mon Sep 17 00:00:00 2001 From: Ben Capodanno Date: Wed, 11 Sep 2024 16:56:15 -0700 Subject: [PATCH] Add handling for NA values during CSV validation --- src/mavedb/lib/mave/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index f59a3fca..dd6b7591 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -1,5 +1,7 @@ import re +import pandas as pd + NA_VALUE = "NA" NULL_VALUES = ("", "na", "nan", "nil", "none", "null", "n/a", "undefined", NA_VALUE) @@ -22,6 +24,9 @@ def is_csv_null(value): """Return True if a string from a CSV file represents a NULL value.""" + # Avoid any boolean miscasts from comparisons by handling NA types up front. + if pd.isna(value): + return True # Number 0 is treated as False so that all 0 will be converted to NA value. if value == 0: return value