From 314e1c7002ec83fb52f55df498317c5c12dabc1c Mon Sep 17 00:00:00 2001 From: larsevj Date: Mon, 20 Jan 2025 17:38:51 +0100 Subject: [PATCH] Try pass in distance array to linkage method --- src/ert/analysis/misfit_preprocessor.py | 10 ++++++---- tests/ert/unit_tests/analysis/test_es_update.py | 1 - .../unit_tests/analysis/test_misfit_preprocessor.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ert/analysis/misfit_preprocessor.py b/src/ert/analysis/misfit_preprocessor.py index ea8ed6f1998..fd3c15a7ea7 100644 --- a/src/ert/analysis/misfit_preprocessor.py +++ b/src/ert/analysis/misfit_preprocessor.py @@ -56,10 +56,12 @@ def cluster_responses( Cluster responses using hierarchical clustering based on Spearman correlation. Observations that tend to vary similarly across different simulation runs will be clustered together. """ - correlation = spearmanr(responses).statistic - if isinstance(correlation, np.float64): - correlation = np.array([[1, correlation], [correlation, 1]]) - linkage_matrix = linkage(correlation, "average", "euclidean") + distance = 1 - np.abs(spearmanr(responses).statistic) + if isinstance(distance, np.float64): + distance = np.array(distance) + linkage_matrix = linkage( + distance[np.triu_indices(distance.shape[0], k=1)], "average", "euclidean" + ) return fcluster(linkage_matrix, nr_clusters, criterion="maxclust", depth=2) diff --git a/tests/ert/unit_tests/analysis/test_es_update.py b/tests/ert/unit_tests/analysis/test_es_update.py index 2101a97abca..5b080c3a988 100644 --- a/tests/ert/unit_tests/analysis/test_es_update.py +++ b/tests/ert/unit_tests/analysis/test_es_update.py @@ -59,7 +59,6 @@ def obs() -> polars.DataFrame: @pytest.mark.integration_test -@pytest.mark.flaky(reruns=5) @pytest.mark.parametrize( "misfit_preprocess", [[["*"]], [], [["FOPR"]], [["FOPR"], ["WOPR_OP1_1*"]]] ) diff --git a/tests/ert/unit_tests/analysis/test_misfit_preprocessor.py b/tests/ert/unit_tests/analysis/test_misfit_preprocessor.py index 80397364bb8..f1cbc35b501 100644 --- a/tests/ert/unit_tests/analysis/test_misfit_preprocessor.py +++ b/tests/ert/unit_tests/analysis/test_misfit_preprocessor.py @@ -68,11 +68,11 @@ def test_misfit_preprocessor(nr_observations): nr_realizations = 1000 Y = np.ones((nr_observations, nr_realizations)) parameters_a = rng.standard_normal(nr_realizations) - parameters_b = rng.standard_normal(nr_realizations) + parameters_b = rng.normal(scale=5, size=nr_realizations) for i in range(nr_observations - 1): Y[i] = i + 1 * parameters_a Y[-1] = 5 + 1 * parameters_b - obs_errors = Y.mean(axis=1) + obs_errors = np.array([0.1] * nr_observations) Y_original = Y.copy() obs_error_copy = obs_errors.copy() result, *_ = main(Y, obs_errors)