diff --git a/sdmetrics/single_column/statistical/kscomplement.py b/sdmetrics/single_column/statistical/kscomplement.py index 525e85c7..ef45efb7 100644 --- a/sdmetrics/single_column/statistical/kscomplement.py +++ b/sdmetrics/single_column/statistical/kscomplement.py @@ -1,5 +1,7 @@ """Kolmogorov-Smirnov test based Metric.""" +import sys + import numpy as np import pandas as pd from scipy.stats import ks_2samp @@ -8,6 +10,8 @@ from sdmetrics.single_column.base import SingleColumnMetric from sdmetrics.utils import is_datetime +MAX_DECIMALS = sys.float_info.dig - 1 + class KSComplement(SingleColumnMetric): """Kolmogorov-Smirnov statistic based metric. @@ -57,6 +61,9 @@ def compute(real_data, synthetic_data): real_data = pd.to_numeric(real_data) synthetic_data = pd.to_numeric(synthetic_data) + real_data = real_data.round(MAX_DECIMALS) + synthetic_data = synthetic_data.round(MAX_DECIMALS) + try: statistic, _ = ks_2samp(real_data, synthetic_data) except ValueError as e: diff --git a/tests/integration/reports/multi_table/test_quality_report.py b/tests/integration/reports/multi_table/test_quality_report.py index fb571e81..8e372459 100644 --- a/tests/integration/reports/multi_table/test_quality_report.py +++ b/tests/integration/reports/multi_table/test_quality_report.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd +from packaging import version from sdmetrics.demos import load_demo from sdmetrics.reports.multi_table.quality_report import QualityReport @@ -299,6 +300,15 @@ def test_quality_report_with_errors(): 'Property': ['Column Shapes', 'Column Pair Trends', 'Cardinality', 'Intertable Trends'], 'Score': [0.8165079365079364, 0.55, 0.95, 0.5833333333333334], }) + + pandas_version = version.parse(pd.__version__) + if pandas_version >= version.parse('2.2.0'): + err1 = "TypeError: '<' not supported between instances of 'int' and 'str'" + err2 = "TypeError: '<' not supported between instances of 'Timestamp' and 'str'" + err3 = "TypeError: '<' not supported between instances of 'float' and 'str'" + else: + err1 = err2 = err3 = "TypeError: can't multiply sequence by non-int of type 'float'" + expected_details = pd.DataFrame({ 'Table': [ 'users', @@ -334,11 +344,11 @@ def test_quality_report_with_errors(): 'Error': [ None, None, - "TypeError: '<' not supported between instances of 'int' and 'str'", + err1, np.nan, np.nan, - "TypeError: '<' not supported between instances of 'Timestamp' and 'str'", - "TypeError: '<' not supported between instances of 'float' and 'str'", + err2, + err3, None, ], }) diff --git a/tests/integration/reports/single_table/_properties/test_column_shapes.py b/tests/integration/reports/single_table/_properties/test_column_shapes.py index 5d490027..748e9db2 100644 --- a/tests/integration/reports/single_table/_properties/test_column_shapes.py +++ b/tests/integration/reports/single_table/_properties/test_column_shapes.py @@ -1,4 +1,5 @@ import pandas as pd +from packaging import version from sdmetrics.demos import load_demo from sdmetrics.reports.single_table._properties import ColumnShapes @@ -84,15 +85,22 @@ def test_get_score_errors(self): # Run column_shape_property = ColumnShapes() - - expected_message_1 = ( - "TypeError: '<' not supported between instances of 'Timestamp' and 'int'" - ) - expected_message_2 = "TypeError: '<' not supported between instances of 'str' and 'float'" - score = column_shape_property.get_score(real_data, synthetic_data, metadata) # Assert + pandas_version = version.parse(pd.__version__) + if pandas_version >= version.parse('2.2.0'): + expected_message_1 = ( + "TypeError: '<' not supported between instances of 'Timestamp' and 'int'" + ) + expected_message_2 = ( + "TypeError: '<' not supported between instances of 'str' and 'float'" + ) + else: + expected_message_1 = ( + "TypeError: unsupported operand type(s) for *: 'Timestamp' and 'float'" + ) + expected_message_2 = "TypeError: can't multiply sequence by non-int of type 'float'" details = column_shape_property.details details_nan = details.loc[pd.isna(details['Score'])] diff --git a/tests/integration/single_column/statistical/test_kscomplement.py b/tests/integration/single_column/statistical/test_kscomplement.py index d15e8bca..d2baf5a0 100644 --- a/tests/integration/single_column/statistical/test_kscomplement.py +++ b/tests/integration/single_column/statistical/test_kscomplement.py @@ -46,3 +46,16 @@ def test_bad(array_like): assert 0.0 <= output < 0.5 assert 0.0 <= normalized < 0.5 + + +def test_one_float_value(): + """Test KSComplement.compute when both data have the same float values GH#652.""" + # Setup + real = pd.Series([0.3 - 0.2]) + synth = pd.Series([0.2 - 0.1]) + + # Run + output = KSComplement.compute(real, synth) + + # Assert + assert output == 1 diff --git a/tests/unit/reports/single_table/_properties/test_column_shapes.py b/tests/unit/reports/single_table/_properties/test_column_shapes.py index f9c688cf..bc84b100 100644 --- a/tests/unit/reports/single_table/_properties/test_column_shapes.py +++ b/tests/unit/reports/single_table/_properties/test_column_shapes.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd +from packaging import version from sdmetrics.reports.single_table._properties.column_shapes import ColumnShapes @@ -108,7 +109,12 @@ def test__generate_details_error(self): result = column_shape_property._generate_details(real_data, synthetic_data, metadata) # Assert - expected_message = "TypeError: '<' not supported between instances of 'str' and 'int'" + pandas_version = version.parse(pd.__version__) + if pandas_version >= version.parse('2.2.0'): + expected_message = "TypeError: '<' not supported between instances of 'str' and 'int'" + else: + expected_message = "TypeError: can't multiply sequence by non-int of type 'float'" + result_nan = result.loc[pd.isna(result['Score'])] column_names_nan = result_nan['Column'].tolist() error_message = result_nan['Error'].tolist()