From 96326ae75600391e927b01bf2e31a93f12915159 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Sat, 9 Dec 2023 22:27:23 +0100 Subject: [PATCH 01/14] Pass progress argument to permutation_montecarlo_shapley inside permutation_montecarlo_shapley --- src/pydvl/value/shapley/common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pydvl/value/shapley/common.py b/src/pydvl/value/shapley/common.py index c4d5db13a..eda884e6e 100644 --- a/src/pydvl/value/shapley/common.py +++ b/src/pydvl/value/shapley/common.py @@ -110,7 +110,13 @@ def compute_shapley_values( ): truncation = kwargs.pop("truncation", NoTruncation()) return permutation_montecarlo_shapley( # type: ignore - u=u, done=done, truncation=truncation, n_jobs=n_jobs, seed=seed, **kwargs + u=u, + done=done, + truncation=truncation, + n_jobs=n_jobs, + seed=seed, + progress=progress, + **kwargs, ) elif mode == ShapleyMode.CombinatorialMontecarlo: return combinatorial_montecarlo_shapley( From dc2d8ec111f6ce010a6b281a185de9fb7c7deb65 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Sat, 9 Dec 2023 22:27:43 +0100 Subject: [PATCH 02/14] Create plot_influence_distribution function --- src/pydvl/reporting/plots.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/pydvl/reporting/plots.py b/src/pydvl/reporting/plots.py index 4e8e5afa5..7c0f19b73 100644 --- a/src/pydvl/reporting/plots.py +++ b/src/pydvl/reporting/plots.py @@ -270,6 +270,26 @@ def plot_shapley( return ax +def plot_influence_distribution( + influences: NDArray[np.float_], index: int, title_extra: str = "" +) -> plt.Axes: + """Plots the histogram of the influence that all samples in the training set + have over a single sample index. + + Args: + influences: array of influences (training samples x test samples) + index: Index of the test sample for which the influences + will be plotted. + title_extra: Additional text that will be appended to the title. + """ + _, ax = plt.subplots() + ax.hist(influences[:, index], alpha=0.7) + ax.set_xlabel("Influence values") + ax.set_ylabel("Number of samples") + ax.set_title(f"Distribution of influences {title_extra}") + return ax + + def plot_influence_distribution_by_label( influences: NDArray[np.float_], labels: NDArray[np.float_], title_extra: str = "" ): @@ -279,7 +299,7 @@ def plot_influence_distribution_by_label( Args: influences: array of influences (training samples x test samples) labels: labels for the training set. - title_extra: + title_extra: Additional text that will be appended to the title. """ _, ax = plt.subplots() unique_labels = np.unique(labels) @@ -287,6 +307,6 @@ def plot_influence_distribution_by_label( ax.hist(influences[labels == label], label=label, alpha=0.7) ax.set_xlabel("Influence values") ax.set_ylabel("Number of samples") - ax.set_title(f"Distribution of influences " + title_extra) + ax.set_title(f"Distribution of influences {title_extra}") ax.legend() plt.show() From 7fd475055cb349b23b4db4f4c5f54d70b3ee6306 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Sat, 9 Dec 2023 22:28:14 +0100 Subject: [PATCH 03/14] Improve readme, add plots of readme examples' results --- README.md | 370 +++++--- docs/assets/data_valuation_example.svg | 876 +++++++++++++++++ docs/assets/influence_functions_example.svg | 993 ++++++++++++++++++++ 3 files changed, 2107 insertions(+), 132 deletions(-) create mode 100644 docs/assets/data_valuation_example.svg create mode 100644 docs/assets/influence_functions_example.svg diff --git a/README.md b/README.md index 57bf56d33..101516860 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,242 @@

-pyDVL collects algorithms for Data Valuation and Influence Function computation. +**pyDVL** collects algorithms for **Data Valuation** and **Influence Function** computation. -Data Valuation is the task of estimating the intrinsic value of a data point -wrt. the training set, the model and a scoring function. We currently implement -methods from the following papers: +**Data Valuation** is the task of estimating the intrinsic value of a data point +wrt. the training set, the model and a scoring function. + +**Influence Functions** compute the effect that single points have on an estimator / +model + +# Installation + +To install the latest release use: + +```shell +$ pip install pyDVL +``` + +You can also install the latest development version from +[TestPyPI](https://test.pypi.org/project/pyDVL/): + +```shell +pip install pyDVL --index-url https://test.pypi.org/simple/ +``` + +pyDVL has also extra dependencies for certain functionalities (e.g. influence functions). + +For more instructions and information refer to [Installing pyDVL +](https://pydvl.org/stable/getting-started/installation/) in the +documentation. + +# Usage + +In the following subsections, we will showcase the usage of pyDVL +for Data Valuation and Influence Functions using simple examples. + +For more instructions and information refer to [Getting +Started](https://pydvl.org/stable/getting-started/first-steps/) in +the documentation. +We provide several examples for data valuation +(e.g. [Shapley Data Valuation](https://pydvl.org/stable/examples/shapley_basic_spotify/)) +and for influence functions +(e.g. [Influence Functions for Neural Networks](https://pydvl.org/stable/examples/influence_imagenet/)) +with details on the algorithms and their applications. + +## Influence Functions + +For influence computation, follow these steps: + +1. Import the necessary packages (The exact packages depend on your specific use case). + + ```python + import torch + from torch import nn + from torch.utils.data import DataLoader, TensorDataset + from pydvl.reporting.plots import plot_influence_distribution + from pydvl.influence import compute_influences, InversionMethod + from pydvl.influence.torch import TorchTwiceDifferentiable + ``` + +2. Create PyTorch data loaders for your train and test splits. + + ```python + torch.manual_seed(16) + + input_dim = (5, 5, 5) + output_dim = 3 + + train_data_loader = DataLoader( + TensorDataset(torch.rand((10, *input_dim)), torch.rand((10, output_dim))), + batch_size=2, + ) + test_data_loader = DataLoader( + TensorDataset(torch.rand((5, *input_dim)), torch.rand((5, output_dim))), + batch_size=1, + ) + ``` + +3. Instantiate your neural network model. + + ```python + nn_architecture = nn.Sequential( + nn.Conv2d(in_channels=5, out_channels=3, kernel_size=3), + nn.Flatten(), + nn.Linear(27, 3), + ) + nn_architecture.eval() + ``` + +4. Define your loss: + + ```python + loss = nn.MSELoss() + ``` + +5. Wrap your model and loss in a `TorchTwiceDifferentiable` object. + + ```python + model = TorchTwiceDifferentiable(nn_architecture, loss) + ``` + +6. Compute influence factors by providing training data and inversion method. + Using the conjugate gradient algorithm, this would look like: + + ```python + influences = compute_influences( + model, + training_data=train_data_loader, + test_data=test_data_loader, + inversion_method=InversionMethod.Cg, + hessian_regularization=1e-1, + maxiter=200, + progress=True, + ) + ``` + The result is a tensor of shape `(training samples x test samples)` + that contains at index `(i, j`) the influence of training sample `i` on + test sample `j`. + +7. Visualize the results. + + ```python + plot_influence_distribution(influences, index=1, title_extra="Example") + ``` + + ![Influence Functions Example](docs/assets/influence_functions_example.svg) + + The higher the absolute value of the influence of a training sample + on a test sample, the more influential it is for the chosen test sample, model + and data loaders. The sign of the influence determines whether it is + useful (positive) or harmful (negative). + +> **Note** pyDVL currently only support PyTorch for Influence Functions. +> We are planning to add support for Jax and perhaps TensorFlow or even Keras. + +## Data Valuation + +The steps required to compute data values for your samples are: + +1. Import the necessary packages (The exact packages depend on your specific use case). + + ```python + import matplotlib.pyplot as plt + from sklearn.datasets import load_breast_cancer + from sklearn.linear_model import LogisticRegression + from pydvl.reporting.plots import plot_shapley + from pydvl.utils import Dataset, Scorer, Utility + from pydvl.value import ( + compute_shapley_values, + ShapleyMode, + MaxUpdates, + ) + ``` + +2. Create a `Dataset` object with your train and test splits. + + ```python + data = Dataset.from_sklearn( + load_breast_cancer(), + train_size=10, + stratify_by_target=True, + random_state=16, + ) + ``` + +3. Create an instance of a `SupervisedModel` (basically any sklearn compatible + predictor). + + ```python + model = LogisticRegression() + ``` + +4. Create a `Utility` object to wrap the Dataset, the model and a scoring + function. + + ```python + u = Utility( + model, + data, + Scorer("accuracy", default=0.0) + ) + ``` + +5. Use one of the methods defined in the library to compute the values. + In our example, we will use *Permutation Montecarlo Shapley*, + an approximate method for computing Data Shapley values. + + ```python + values = compute_shapley_values( + u, + mode=ShapleyMode.PermutationMontecarlo, + done=MaxUpdates(100), + seed=16, + progress=True + ) + ``` + The result is a variable of type `ValuationResult` that contains + the indices and their values as well as other attributes. + +6. Convert the valuation result to a dataframe and visualize the values. + + ```python + df = values.to_dataframe(column="data_value") + plot_shapley(df, title="Data Valuation Example", xlabel="Index", ylabel="Value") + plt.show() + ``` + + ![Data Valuation Example Plot](docs/assets/data_valuation_example.svg) + + The higher the value for an index, the more important it is for the chosen + model, dataset and scorer. + +## Caching + +pyDVL offers the possibility to cache certain results and +speed up computation. It uses [Memcached](https://memcached.org/) For that. + +You can run it either locally or, using +[Docker](https://www.docker.com/): + +```shell +docker container run --rm -p 11211:11211 --name pydvl-cache -d memcached:latest +``` + +You can read more in the +[documentation](https://pydvl.org/stable/getting-started/first-steps/#caching). + +# Contributing + +Please open new issues for bugs, feature requests and extensions. You can read +about the structure of the project, the toolchain and workflow in the [guide for +contributions](CONTRIBUTING.md). + +# Papers + +## Data Valuation + +We currently implement the following papers: - Castro, Javier, Daniel Gómez, and Juan Tejada. [Polynomial Calculation of the Shapley Value Based on Sampling](https://doi.org/10.1016/j.cor.2008.04.004). @@ -80,8 +311,9 @@ methods from the following papers: Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). New Orleans, Louisiana, USA, 2022. -Influence Functions compute the effect that single points have on an estimator / -model. We implement methods from the following papers: +## Influence Functions + +We currently implement the following papers: - Koh, Pang Wei, and Percy Liang. [Understanding Black-Box Predictions via Influence Functions](http://proceedings.mlr.press/v70/koh17a.html). In @@ -94,132 +326,6 @@ model. We implement methods from the following papers: [Scaling Up Influence Functions](http://arxiv.org/abs/2112.03052). In Proceedings of the AAAI-22. arXiv, 2021. -# Installation - -To install the latest release use: - -```shell -$ pip install pyDVL -``` - -You can also install the latest development version from -[TestPyPI](https://test.pypi.org/project/pyDVL/): - -```shell -pip install pyDVL --index-url https://test.pypi.org/simple/ -``` - -For more instructions and information refer to [Installing pyDVL -](https://pydvl.org/stable/getting-started/installation/) in the -documentation. - -# Usage - -### Influence Functions - -For influence computation, follow these steps: - -1. Wrap your model and loss in a `TorchTwiceDifferentiable` object -2. Compute influence factors by providing training data and inversion method - -Using the conjugate gradient algorithm, this would look like: -```python -import torch -from torch import nn -from torch.utils.data import DataLoader, TensorDataset - -from pydvl.influence import TorchTwiceDifferentiable, compute_influences, InversionMethod - -nn_architecture = nn.Sequential( - nn.Conv2d(in_channels=5, out_channels=3, kernel_size=3), - nn.Flatten(), - nn.Linear(27, 3), -) -loss = nn.MSELoss() -model = TorchTwiceDifferentiable(nn_architecture, loss) - -input_dim = (5, 5, 5) -output_dim = 3 - -train_data_loader = DataLoader( - TensorDataset(torch.rand((10, *input_dim)), torch.rand((10, output_dim))), - batch_size=2, -) -test_data_loader = DataLoader( - TensorDataset(torch.rand((5, *input_dim)), torch.rand((5, output_dim))), - batch_size=1, -) - -influences = compute_influences( - model, - training_data=train_data_loader, - test_data=test_data_loader, - progress=True, - inversion_method=InversionMethod.Cg, - hessian_regularization=1e-1, - maxiter=200, -) -``` - - -### Shapley Values -The steps required to compute values for your samples are: - -1. Create a `Dataset` object with your train and test splits. -2. Create an instance of a `SupervisedModel` (basically any sklearn compatible - predictor) -3. Create a `Utility` object to wrap the Dataset, the model and a scoring - function. -4. Use one of the methods defined in the library to compute the values. - -This is how it looks for *Truncated Montecarlo Shapley*, an efficient method for -Data Shapley values: - -```python -from sklearn.datasets import load_breast_cancer -from sklearn.linear_model import LogisticRegression -from pydvl.value import * - -data = Dataset.from_sklearn(load_breast_cancer(), train_size=0.7) -model = LogisticRegression() -u = Utility(model, data, Scorer("accuracy", default=0.0)) -values = compute_shapley_values( - u, - mode=ShapleyMode.TruncatedMontecarlo, - done=MaxUpdates(100) | AbsoluteStandardError(threshold=0.01), - truncation=RelativeTruncation(u, rtol=0.01), -) -``` - -For more instructions and information refer to [Getting -Started](https://pydvl.org/stable/getting-started/first-steps/) in -the documentation. We provide several examples for data valuation -(e.g. [Shapley Data Valuation](https://pydvl.org/stable/examples/shapley_basic_spotify/)) -and for influence functions -(e.g. [Influence Functions for Neural Networks](https://pydvl.org/stable/examples/influence_imagenet/)) -with details on the algorithms and their applications. - -## Caching - -pyDVL offers the possibility to cache certain results and -speed up computation. It uses [Memcached](https://memcached.org/) For that. - -You can run it either locally or, using -[Docker](https://www.docker.com/): - -```shell -docker container run --rm -p 11211:11211 --name pydvl-cache -d memcached:latest -``` - -You can read more in the -[documentation](https://pydvl.org/stable/getting-started/first-steps/#caching). - -# Contributing - -Please open new issues for bugs, feature requests and extensions. You can read -about the structure of the project, the toolchain and workflow in the [guide for -contributions](CONTRIBUTING.md). - # License pyDVL is distributed under diff --git a/docs/assets/data_valuation_example.svg b/docs/assets/data_valuation_example.svg new file mode 100644 index 000000000..21c0f885d --- /dev/null +++ b/docs/assets/data_valuation_example.svg @@ -0,0 +1,876 @@ + + + + + + + + 2023-12-09T21:47:38.055137 + image/svg+xml + + + Matplotlib v3.7.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/influence_functions_example.svg b/docs/assets/influence_functions_example.svg new file mode 100644 index 000000000..a7040e1c3 --- /dev/null +++ b/docs/assets/influence_functions_example.svg @@ -0,0 +1,993 @@ + + + + + + + + 2023-12-09T22:22:51.558936 + image/svg+xml + + + Matplotlib v3.7.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From fb3ae87bed46c5cea0e7a2f22faf3ddf46738d65 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Sat, 9 Dec 2023 22:33:31 +0100 Subject: [PATCH 04/14] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2a06c7ec..71ad47442 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Changed +- Improve readme and explain better the examples + [PR #465](https://github.com/aai-institute/pyDVL/pull/465) - Simplify and improve tests, add CodeCov code coverage [PR #429](https://github.com/aai-institute/pyDVL/pull/429) From 9bb6498bf69b0eb8be8901966cf9ac8b8ff24e69 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 13:06:57 +0100 Subject: [PATCH 05/14] Apply suggestions from code review Co-authored-by: Miguel de Benito Delgado --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 101516860..7d06353ef 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,16 @@ **pyDVL** collects algorithms for **Data Valuation** and **Influence Function** computation. -**Data Valuation** is the task of estimating the intrinsic value of a data point -wrt. the training set, the model and a scoring function. - -**Influence Functions** compute the effect that single points have on an estimator / -model +**Data Valuation** for machine learning is the task of assigning a scalar +to each element of a training set which reflects its contribution to the final +performance or outcome of some model trained on it. Some concepts of +value depend on a specific model of interest, while others are model-agnostic. +pyDVL focuses on model-dependent methods. + +The **Influence Function** is an infinitesimal measure of the effect that single +training points have over the parameters of a model, or any function thereof. +In particular, in machine learning they are also used to compute the effect +of training samples over individual test points. # Installation From 257f9e4df2827cbc81186d69fe65b24f3b853a48 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 13:28:56 +0100 Subject: [PATCH 06/14] Remove ugly plots from readme --- README.md | 22 +- docs/assets/data_valuation_example.svg | 876 ----------------- docs/assets/influence_functions_example.svg | 993 -------------------- 3 files changed, 4 insertions(+), 1887 deletions(-) delete mode 100644 docs/assets/data_valuation_example.svg delete mode 100644 docs/assets/influence_functions_example.svg diff --git a/README.md b/README.md index 7d06353ef..aa29a3516 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,6 @@ For influence computation, follow these steps: import torch from torch import nn from torch.utils.data import DataLoader, TensorDataset - from pydvl.reporting.plots import plot_influence_distribution from pydvl.influence import compute_influences, InversionMethod from pydvl.influence.torch import TorchTwiceDifferentiable ``` @@ -152,14 +151,6 @@ For influence computation, follow these steps: that contains at index `(i, j`) the influence of training sample `i` on test sample `j`. -7. Visualize the results. - - ```python - plot_influence_distribution(influences, index=1, title_extra="Example") - ``` - - ![Influence Functions Example](docs/assets/influence_functions_example.svg) - The higher the absolute value of the influence of a training sample on a test sample, the more influential it is for the chosen test sample, model and data loaders. The sign of the influence determines whether it is @@ -178,7 +169,6 @@ The steps required to compute data values for your samples are: import matplotlib.pyplot as plt from sklearn.datasets import load_breast_cancer from sklearn.linear_model import LogisticRegression - from pydvl.reporting.plots import plot_shapley from pydvl.utils import Dataset, Scorer, Utility from pydvl.value import ( compute_shapley_values, @@ -232,18 +222,14 @@ The steps required to compute data values for your samples are: The result is a variable of type `ValuationResult` that contains the indices and their values as well as other attributes. -6. Convert the valuation result to a dataframe and visualize the values. + The higher the value for an index, the more important it is for the chosen + model, dataset and scorer. + +6. (Optional) Convert the valuation result to a dataframe and analyze and visualize the values. ```python df = values.to_dataframe(column="data_value") - plot_shapley(df, title="Data Valuation Example", xlabel="Index", ylabel="Value") - plt.show() ``` - - ![Data Valuation Example Plot](docs/assets/data_valuation_example.svg) - - The higher the value for an index, the more important it is for the chosen - model, dataset and scorer. ## Caching diff --git a/docs/assets/data_valuation_example.svg b/docs/assets/data_valuation_example.svg deleted file mode 100644 index 21c0f885d..000000000 --- a/docs/assets/data_valuation_example.svg +++ /dev/null @@ -1,876 +0,0 @@ - - - - - - - - 2023-12-09T21:47:38.055137 - image/svg+xml - - - Matplotlib v3.7.2, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/assets/influence_functions_example.svg b/docs/assets/influence_functions_example.svg deleted file mode 100644 index a7040e1c3..000000000 --- a/docs/assets/influence_functions_example.svg +++ /dev/null @@ -1,993 +0,0 @@ - - - - - - - - 2023-12-09T22:22:51.558936 - image/svg+xml - - - Matplotlib v3.7.2, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From e5c757b6545e1f3c98fa6578de295325008e2547 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 13:54:51 +0100 Subject: [PATCH 07/14] Use better images in readme --- README.md | 24 ++++++++++++++++++++ docs/assets/influence_functions_example.png | Bin 0 -> 55322 bytes 2 files changed, 24 insertions(+) create mode 100644 docs/assets/influence_functions_example.png diff --git a/README.md b/README.md index aa29a3516..fb45f342b 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,35 @@ performance or outcome of some model trained on it. Some concepts of value depend on a specific model of interest, while others are model-agnostic. pyDVL focuses on model-dependent methods. +
+ best sample removal +
+ Comparison of different data valuation methods +
on best sample removal. +
+
+ The **Influence Function** is an infinitesimal measure of the effect that single training points have over the parameters of a model, or any function thereof. In particular, in machine learning they are also used to compute the effect of training samples over individual test points. +
+ best sample removal +
+ Influences of input points with corrupted data. +
Highlighted points have flipped labels. +
+
+ # Installation To install the latest release use: diff --git a/docs/assets/influence_functions_example.png b/docs/assets/influence_functions_example.png new file mode 100644 index 0000000000000000000000000000000000000000..94f804e9ebe47e08928882ebb8fc9cdbf7f56457 GIT binary patch literal 55322 zcmeFYby!s2zdt%b0 zbvO9=p7TBTp8Lmnp8I>A`^P~y&&=Lyz2mjs>s@OTucxC1BcLMyfj}^Xx{5vsgf#&I zL0a%|fIA22Uy4B>3jbh3Q(t|X02VKAPe-Ji1B-8vmjjDKAkq;83Y>nG?d1JKCM5RK zj?xn1ZAxUD8Gu?FuVt!sL`x|4pt^x+4SmC8VpN2@R}KAdr+X4j_vM9bHYdrfj@oT7y=%`RbjB<5 z?}eU5@T+)oe#D>O!9*>`ooYE;vg4+a9B-^cDPySaWUkZZ(o3`I z)}_O+JP(ncwWhsT$$4Go?e}z#J;QO9h1O>nt&|9F8hYRIoVMF#_IVI6eA_Si^J559I)!3*;P2tCEm+Ds&(=MJJ*tbO+W;I?{HO$k#^;sPn&m#|J z4v!&K6JbvlS$8hd?RWb4+%ex_;+vyYcr12!Wm|v6>#B~qk_`V@LQbs4AB`>XNqjPR{uBeej3_@nG@gHTLB*xXpL4bjOMlCt$wXsc0L!?n6q-+cA5|W zJWU20%&99Sq%@>olOOsSlpt$2Buc(7VX(#+-si_C>Q5N*p8Bn$?n8=C4r7D;I^Pm~ z_zSgs-hRX{G!uwcP7U;Md`G10#^utL<@(Co|9J=#V`Dh1-8gBIl-P&U+bQ{DOjwP* z^>0#qH)pKkl*+Z&((d0N7R)c(dl8lV%r9R}|9I=KR*%f^nO$4_T*iMyn<=bK*Ra~6 z>F{0f_m{bO|5yu~^UimLK?}lL+n0ymhbOZmeD+pd2Rq*#?>DS0n4g{8-=FEhY{k7& z%KqJDn6Kt`>!}3dpv=#=evsy7@((i^^kv4v zXcn2?vM)$KU|7V$5y#2||Mv0mSf<&kMiJ=QZpaJQRs0MKsfwqUeH~nd5ZUxm85nlq z;(@92T^4?y8>Y-k=KP&b0*eTXJ~~*r^o<7HZxkL<4{IU&^+uRom5zjZnt}TdN;Zz` zNWJGs-qZFy2)l6S$7lRH!af55&fnyP_s|~VyDlS|w}u!k%D9D~l?_-Qe2>Yj%M?G2 zQ1d|I^sv7z%ZT1?o#r`tk&d3r%VP8c2anG`F>}fh1x?hh@-?cdPD|&1l?zY~DB29Mw9<9nkj`dwZImJFxU-s-m0Cl)b)eslR7oVJi9_ z1Tz}#lX?Ajr6dK8cO-cgXkMmgz=1q6vdE_5H8(hy}Ydss-d|2j;p z9MzHkI2XCIO(t3Q@X!d?tld%J(4#tbfFB%0zR~EUX!&l=-~mz9w)%|QfLh9>A`31d z5qCk>VSS5*YcJ zj>SU=qP~yAK=}?wEEn+66?VCH-gk3Ba%aHh%f4?|;V|j@u=kbHkC2{6ZjagOoc<1m-KmF1erdf z{f+n>S@FqLrkJS82<$`04 z@9pQ(2u$fnAyjA`h8xrQ?Fw5MU)_m#mrpO#Dw6_Q%*7rE*J#A8c*L=7*9#fCM;~+B zP=Zm{iAp1LJ!?!TfN}b+Vqz`k)mo1MPSy%ac9YtWn9}VfS;Y_DEgh3w3T*ZS1hT}K z_pZGawAuyjSZDpFqV`KRsgD;l&^yANWpR(-;mt>C23G1U%NJs>k9ERgG*n72n7!yz zrk{*9zt0&{4Tk@SWwBpOWR*We;-mL?c9{kTuwGC5{OWdM)F?ZHa-KLJIQj)yj!IY1 zQ0gdZr-%-qwX&x^4U}=DK(}8?I;vuhP-jeHGXu7IU1F)z)+pukH@ANX?fnu+_8?xt zz5CUj@f%AQp;~XBz>Uoj#e0P%GZ#rh`r98)@jess(30hGy(QniyT7$7`NC+e^foPn z7CKeppg%=nuTFot2lm^-vXHj94UaEtY)VC_K7s#%kDv@R5Bmj-AOB7`l}kZ3UWSum z#@J$ck<=aSEw3Wwcgkf=HApVhM$)8Iq19~4w)U;D$35AuuYktE2KqNk-g=K+Md@i37xxVc|;4_*ZDgOBh)?@$E62Vy3 zo8Rh-C=4Ifz`QgJvwOYVeR-#5stG+J#mHRsqJDA~GYb-dh2e>%Bc5Vbsn#}CkJysy zqjGf)2@kXDy(*05l6mRu`AK|IM5&hWJ8$ReFvGM;#CJ7ysGeHBiskl=nt$KQ$=yw8 z{4J^I4r@y!wf5J@{)x@+(SL+>65ll0gB9hN`jljEKX|b-ivzVTH>s4$_od>K#1d(w zx{ca-NaN1s#S&eM6_y&tOlkSMjIx5^EHGeQ)A3?Db%j^}>8T*IRotl%(UnA7$iia-|MsQgNOazFTC4gv#DTLA97%;|8is2 zwt|J$%S}AvsPa+dtxO$k_OGB-F-tRHnN~KK2ksByRG1#Y`SftAS`ks%gJn`N`tngi^nUd1_)Tbe_ zBjY4Z)tmXb$$4i1pj@WqcA^jXEGbll8Tf{bTawt=WisPB>{xd+)LOkB&8bGZh}q-0 z+b^;l6If^tw}yRKAj)Gbehp86d6laO!9L(xV`~_r7ZwA;rqVo6QK-IcAQEN+{oGVRom^hMTGc`y!*3%@W&xJOsD6^HgE4r!|F~j z3{qBLDVA};IEp&HtfcU9D1|yTqb>f_<4MrSxlkiq!S`es=c4j+!THOYuC~W`TFQ+< z5p(>0S^ZP5tp|Qm^pl=H`?VGPGuxwQ_?aWC8T=X1y|u%-5zZ zE%3NKQv4XK_$o!fna?SOX^%4Z9jUqKjJxRX2-H&}Ch0wsV)t-{j=MXhvG3N%)%|&@ z)XCxB{un}V=r}24jH22NwtUUVoIdUGLuF%Nobe4~@!ZkDj(Z5(`h%fGXbtIYIgOJzLjd=c*P`0z~Q2 zYB))KO}gasb39aA>@2EqT@YC{Qx87QLmlT4BQLgm+(6NIo#-saM}!U1-+~E66jPrQ z7~r!${92fh($S#AX`a9nWw;u%!iVa|=HC)SJK1sTh40_^4bOUF2Qn3={FFRx{oFT8 zzh_&TkK};tIi0X=-JKTSp!9mXs>J1+h6I-3P4V9Mu^uoYKlAfQKN6Lf`4X}<(OB8c zW-5@|q<7KnMYE{#Oq^%T!F>x-ODY``G+M;#t@DYV%9s0I{uhe})8G2EC{KL(bKku! zq^YYme|Ty!;#sIgN@z*!ymt>pJOo18)L3pLb~M7-Ry4wdFQ3m zpF8=PJzlScr|0gKn}L{X<}2U)G;EO`puj}o!7nLNr~>l_mOZdEtedMXBb|E4W~ssh z`d*Hwn(aDbH_FP;H$;yNnfz&uNmTx!ecO!A0og=!y7K^zX2RgG&)$Dy$UaB#R1i^T zuV+>-jYBFfqxgyKAi+hgV8rXc0&LN6cE&)$BS`AKR#CsM1516sDm{a8vcgE}S50CNlT*2d?J#7K=- zqUJ0kNpBtT#>2zc(E}Mg^wx#J^=RG479_hlj+de45iuzL=y)Hv7U~Xm3!W;|wDcBi zI1*c%^%g5fmui=n>*Y?{=k@W}%eL&voVfNZG7bv2YOAkD;SDZnt;Al1^-um}=i(21 zNhp5nUuBpVz)Y6-E(@OjW}oPhvK>nXsj`>gg}Wia5MwQ&Zz&E^#+F|Diz-8HE}2FF zW?Gu4;!YCr*gM;?2@cKi6$c-#v#L)X@z1SXv>FlOhr02sSSfyqbT8e$R}~i0jW)6u zyu4MRxNPm=JRPdw*^)D%XBy+b+8UG5C_)2MrsC0AQwcYpnIX(x5>;{-A$+9rTapk$ zV=zucUYzf){FR(%K<|b0a{|aqN`B|@{d=+Wo?pv^>J9uHHA6#mS4q8PAm*SYcO^M3 zo7kgV)Qzzu8;3kKdU0=&*Dg7?yadjZM$#;wOlunOyvS(fjxzYdek;V&JXCHvuzf7B zo;s)`c7f;1)BY-~`w6sn%l+37&4>4+CGhOzoCyZ8?PTlZtH-J+X9?5{r`ocmg$ciPJgq7;w8>dX7>+F#LfB1?4>hwhFM4@o$(^S<&O zY2V+ljzvM|a>}G7m)1}cM2*M4^9N7*Xp zm%d}ctocenS*)K0pmkg{6FbQ*t7)mvA0|e1?Wi`EALHdk3bA6LW=j^V%Ay?4aK`F33BsupAU1C!PwG$jCD?CI&qUW_UVjjKFPWS#d7rj7L10BL zw?HDhTq&?>iTB&mHZyPJb|20dn7s|m30=G6B@7iMlq(s!V@cq#T{Xy|rxW6#-h}Pz zS|fQXD-&l7TM2!u95SPLILmr+;3ZtrASY;YJOAYcX84C%nPlq5XaW0$P+*9(=UYcA z9uNo|gH%@5Lntf%s{;x29dpA{<jGiG2|wEd558a;7%EdORO=Y zmgOH@ZMrlHYPI(b7|gKk`1_uFCb%YwBxm3Ueh>70Q^|HT2?T6Iy@Hrm+kKjMOsM7wms~wsZdO&sc5`3Z*Xl z`NmhSd6xY7df3qE2OHxIqwhZ7AEeW9g#T#yzDK_r^%*jJBj@Ls?1L;*{`$JnkCo0Y zAB4X@s`!~9WG`zh?fGD$-)Lepy%V1C)#T|r9q}kEsyZxuHVVVLO0*I$>oPH!55io2c~pRu~0Z>KFhDlEXe+yru5*&e$NAW)yQg#7k7PfckFtRxuEw> z7!A*pn|{_S>A&x5K4AbuXV5)gJ7V$7tk>Vfn*f|y=?4U9ei1A9h{IJa%?-Toop;fdpR}}ac#J^m$HL1Qa#w) z!5~=2&@R~3PTHPLL7qT1PzE62?%->~66o&c;Ug0$$97Fu2Kan+TZoP2dWf&99Gj`O z9*eT4w*!lqpqL(bg05C$pC&q7bt zKAwKwb`E#^9Xx#5|DA-r-G9vY^7D4PR>$5>$idCQ9T@5Za25V9EY%R&djB!w3IZpj zyVvzB0PO!V(iiFYpJ4qL*{+^ktMl)I0Mq}2?tdBmFWs*P1EaLHWmG)v{I0}9sK~Kh z@t3jpv_sm5EB-Y6p#`Y7Z-38v9Ym}kdlN;Nl5=2DTIfQ zuZ@SD!xbrjxF8asBLTO!6SWl+7ZA0xwHFYx7l#YjI7k6>;I{T6wl;7PJGjKZk! z)zjPE23Ss{yN#2Bke7$k^@A(GWfb)ga%>`k@c+D{=Vs&U2uzS;(?WXq1^(v)L!`Td zfv?RKn!=KB5fN!=5n(AQX)#H$zcOFbGj{Oy0W9%~rZ8Mk^e^43Wsw1t0f4o+vQvP- zHCaFx8D(z=8(&XvLr+gPIkqbjELStH&6`E`@1m$9eSi@`R~G+|%^Ns8`1|ee7H~se zKVo6IHd`4RyT43)Z2TSUuNeXJ{ywsEw()Rs0QUDk2KBG|$p2=s>>L~&0F2=Ra61uc z0WoQ5xPX+6jhKL~E%4jU(O%p^Sn9u0_wjV}4Y2WcP;>$`1+)PS=vo^Vo@*$1|Lf=g zXNN1C;3A>|!Xg5~!iK`4GUB2#qEa{E!ZL6;o6tWB3tjE%e`zc$^#8y__Ikj-xdC9_ z-+O?20lbyaf4HlEFuQ{B|KQI*Yw>@O13>zJnf#CN{a?8L7q0&i0{^4I|7%_U3)lY$ zf&bCq|Fy3F&)_2X&w|In1IU5`fO2M_Z$ttpg>Y;&)l@*2SN~qLm!<=E@V(UU`G7#g zHdhz8OraDB+{E=oXy3tIy+L?GoU2mFr~w3G0U=Zr4Fjh)=R$IGm%gF*7Tk!U@RX+d zwujh$XveyX8P!+d_P#9RtD*99X&?%_yUMhN-}B^UgsB1-}dcx>0jaqW%o@N{EyYfQWyKFbP{LHkD>o2mfC`#AAD06aYsyr>j{J%bY>PE_Ba8hTh z{dYJW7WqFDM_HsO4XEbA-4ZEi zUCOW>Ah^p!0^Wg?2Gglu3(RY^oNIbY<0ZGtiRp|aMZ2SA&SzKQ7#a+dwFw3vBW6vk zi+#<(knhc%>5KwzOFMU^n)A}gRa6*?@U(gO2qhg;OowJf&!C4VyF{xfp7kkSLlkQ{?84rmpRvf8*1!FrK0`Np?_**>A}9g zWhSr_wpZns@DQyIX70~>9Jw)OXiD_L(klC3Bq}2MR0~eu4);W!3~QqQ zprvE{@OnwIz}b_Q*UJHjiRm@iDH3eX8o zt({rgK#*zTh(JA^ETI^v6N^o$XL>MZyR0Zo`g&Om>HcD^Q>l$jh8DtetrVCUS{HVd zg%|c-)I{#181egx1tPz$buKD$c1R>+l|X&I+ueftL^6gV{x0mYqdjfJPCUcM%6GYT zy&ugQLegqGW?!KdWBS`$Q8}pg$X9XKBa6R(N_%mubgXwozT}taD2d_81_co-$K*DP z;c-~}(WGU_ClUaS72k7pYZGffOe8Xq5n05BNHe6hP;B_l#IE(cU4GsQ8c5PSS zygr^=U4zt_w1Ty1mbr8i1U5)pT!#9zgufvLpcx-3j(^$*IfWEp&p%Gy(lmYOPT0+k z&^KzKmO>E^duQE{oy)DZS^!vzX)%u$eRu0mju$^TWT}-{tPn%rj~S|X7Sy4hW+<2a z^K2yA{)Sgs0ciePC$@MAOU|-p~0mlmePYBit#8U zT#_771JPRBT@n2GmR&(o!I z84MERrhGN!v2}z}+NO8cow`iaILbYen>|pPAYwA4mE)n%<*)jWNq1E5r4<*h8jk$3 zw$(l{#S3LR=BwqAk4S=Ag1Km!5NR5kSP!9>xFheU_B+wfwx{IlCe3EJ4}-V#8avB! zK4_zZ(S)R8zgtc@(X(Kt@1{lk2>tO^uqoMma4ks68MU_x+38tYz}691<@U7Ij^4Y~ z73`Y);dBG+jz-3*0`^X#$ZlgX7IgY%pAwhTUnUr7`(Yc6gU;}^Jzx*e2n~96>U=B@ z#|dm+Juemaw~y3D4K0VBcwlC}(%te99!=wbCA($t^gk`#k8H=M{^bvy4(ikh4<1N0qh;@v-!qTU(ZL?q3nZV) z9TD)wheM>u1LKGt3po+`V3k@NxC~(cd9~D$tH13yW8@@0^(Vmn~3}rd& z&|7Z%oxzlfikE3gBii#@0wbu*zX{is6#70mTWRNe@T> zCFbS;`;9yf1lxf6>z%qSNA;(}Yn{7VnFkQ0x?dAoQ+(k< z+u}46yjS@K7rL>2o)BcQ0jlm)J)b?~Ma8=JU4^NLM10+KRaeh<_J-JD&T`XoiY zYsh@`9NTStW6Ci!PzL!8URSa>hOY$DmJe)=n?Lv+@nCNR z(2_R}@T00pI2RxWyjh}RJkjy|cu!_xtu^6`Npui8DaPZa8Ke{}`D`ylaJf9G_Cj3_ zIxIQyFpntr(O?$>?_uVl(8;*z4#_+&s29OC296&XOt;k3uc2!~xqaXNsk`JLN;*0BTifke ze*!N`FRL@9xwxa(lOAHgwt(fSsQ(BRi;9jkAgk@3=AUNmct_vNEF}RRRCS%=?5o|P4v zAGaYIqJnPqKG>ne^zHs%9G22;(?~gF$#g*5j&Jc*L1x%zA?nJ|WG#k}0(+kb>%;IiqG4u!lsn z(l|1S$zA!d?_uE9gi1YhXpflJEi-h0gEho2k!-vqYpzYsp@tM>%};r77IN@jyvSwI z)Q*&#H2wT6UOY3cH_|WC6>r>i2^A7aU;8Bvh!dc%^fR4MH_)t#X}}>;srUCdKuEzZ zh6BMT#C6lqG{WiYiLp>Rm=?ZGm*oa*Enw2Jpyo@SoL_df+RK~7F++tB zPmT=GKIor-4PHHmNdnLY5Te}i=9Ctm4F61zq>fbm%Z^X&p39ox?o)B?3`zD9jR!EB z_J)WLr|GC?s2zH`Yk|ZV0A@0 zv`W#?>-!p(0fq({+9$%c6^1^M$Qb)+eQORcLDp}LNsFo1BH!%)kj*!D0tE5lkEhzV zad!oPfDDl2Gy2y2{C1(_a_^&ZI$y9sqyfsqnqv1RrT|SBr}kH!xOj1+bWuPVtAry- zTt}$YRfqzTA^lNC&c|qhd|l6g_?!r>FUe=#(FZfel*2V7|8EHi&e zu`9Dn`Il>&w~ej#_$Ics93}#P+|>@X06hY~rny#9zs}d-uP$T^xNZakkB<~|%PeQ+ z{#lleEYtKTr$`*7rjbW_b^fAWx8T!%xe@Tv|6~1QOGi{*Qt&*zI-FWUIQ7*0KjP$T zRU()VIrIq45oMNUux9#Wa`@=qXB_Z4t^-a&h8v+|fuiz_+=XfG^naI(PDetBhgel> zeZdZmh4~X00mCR_?prBbV`C&q4jo0mL3aSadiL~o;0%Bp;nzl4ym-tq&B#g6;s6Vo z2l&2gWj3X6=A5@845Vltn7p2i81J=5WfZ(ZGhHQDC@e2LIANCKUmH_q)zrQ#0W1q5 z+K1a_tD*J#xnE=inEFr4_0)2iEKmxLCuuD?P_kT^XDVQxwtuHqxN%JsQwE3vvv9wk z2U~M_l>o5^SZ$cUm{&c&tBi8?Ly#a7t^ubQy@Fm2~(#oO>j{3{B{Oq}D zdfa3V+808q(n1hrR2OGN`cqXKG_ zo`1l}2A;iI;sDf03cM6Wn_KJtm2{lbEaoHjy{{@kqPF@-BsN_nVI-a@?J>C*bz$1y zIyn=M+*SW)A7lYsLONjvodO3kPW4@r)u#-FC%9Z;tE$CkA%IrFhycrJiWx=^qSd?E zGdm}=-js-^Gt6Wo{v4SCsPQRxq(0LyqdLZ^#okp7bqna4b}bN<)emwRK`7G z$6);tt$O~$x9DnfdyrlEO+e_APmoh+0rpBy?;&etcjK$Y)c%hyv;sV@ajK!wmI*21@om zAiQDp)46X<{9~tUfv1MwpSOSv*k54SR{NDRwX1`V#mmnGaQw0Tq4TVUHDUD5xDpbw z`iOTt4~0p+UKR3nJk^@}BJjLLGSVcH8+8JF;{L9Z1E^tdt6jm;L@M)eo_a?YD<5Y_ zSWF+8m9`#?6X=h!A=X0pwIk~vzveUu5cFcIbxiKYx0cxr)hzJ{yTXeDK+55sx@kAk z)B5fi-KAg(j)VxncZwiuyw`mge2eDgcbC2~&PcmIv~S3Exvy%qxb(lq&-axXZ=*_% zVW$*jjNy(`<*Vi`uRggA-PN|{8o?KXhUg;v++{h(8^(Aj=k7@-JfBD_H|y5Bwq|cn zOD~GucBx1`XfA&4?C^0B{#TjcQ=XCUKVJ~M1s#j+`e7zx2%hXfPa!j86;m9P?=uWd zsG7y*4TDQo;9OuaFfpwl5Lp;nis)Jh%@rryd+yKo%h|eugdk3rORiD(z&wHL1-h3 zFQEk_0TiyP&c_W)<_JUgNGAlf@{a$El?LYVnN*2~3qj`4^ynrWnp;F=8+GHs(`5-3 zkTIxk&nYuln04+9E|Niy20M%?7h?=pNr-jA=qEq8DR6 zOxK`l4t)yDB9oYA$nz!me0)$ybX|FM;Nh6Zk@`UY-K zXII^7Ps5DT!L^WzRR`_zp|96Q&BsegE9|u&wfr2n4c~2~cWVhX9t`~HUYhGcjE;xC zv?k<;ly8zni(KyU$mVUnTL9SyF&ulp$dHp-9&0ja9$sd^v}(vJW_rm4NAgB8)I6jfI3lvS6E@&W3`3)@D$87<>~D7IAq9m{0i+hawdq$dt}1m{$^Rl6#Wy z4IhFV6ONd-Qh7kl_)?5hR8!DEG8*FC)pC|p90|a+4?=E&FLBQj;n*5{e!a_Z$5+Rm zsH+W7RqL%=xkYHs>vJ={loFLT2C{VanEnOiKYk6NYTh$BO**c=2)&*g3(9 znY319Hw5U+B`n^JcjGUm@BG(1T-ja#N~ucY1GyTq*-D~E;=k>+NpCkVK_{+|IdyYALLn1kb~Y9N2yf+cJ8{D`Y&2B|?V zxBU$0Yfpf0w5kZcYCKgn8;wIPYg`N?#sTB5{Z$q`c$z`T81I%Z#cFUf{UUHT+AH4TizxJbRF%gwld%CS~gH4amSTqey@su*{GgY zOy-zqb+L{j>Kbd-=`Q3uM0EL%IOGLf!ZvYa-JFXTzhomkh|gk|O>%`mT-pdD9kRQb z+yvB)MECXHj(Q$W|BjK1Y-mYVc*T%$IhkkZVA+5JwA~w{?6ZlR47hGf(bk>HUgjPLecWm9dr*Mu2oAY(p$$~{`dSc6R(|me?EGW}gG0#+UQGM0an!5Z zq9b}5y8V)TX~rM)fllr))RVNr`UtuLGNWNas)s>MgHXcSRnP55^~zPYh!grM>nHrs zAE627HPj}SM*--po3NlP zdtXC=8)$sTb)BPS&{caE_xFTaHP7YPVb3kR8Jb=f>c>wLV16J@SAPdA#vpL{%?OXl z<8;!B9|sDxN_f%OARj%6WQHc99m8f<9lPqMm_HOqUs5;o$V-$M=4UYna>6ou(A#@L z0|7vl^^mOOml!P*GS+W(?|z2H7d+Gv8M{yLi4 ziD}(x68z+_`(>+Xcq2#w2AZ$ulh3BM_7gI8*jEIehJJg9YDe8|PfBK` z2z`ay96=VCG{Wf1Zf(PRo(r5lh|N+#3OIeELxe8ceaLo23JDX68MG0=pYPDcnyajJ zoD#7X`ZK_h8Wo=p#tqjm=(9TeZf%uX?r#bSGL!l(AWW&@kf#ZcT6S9gd?O5R{}yEu z&d?dTFp~nnOz}2yh{??AE!%zkSc-FZLX-ljqsZ3eQb+HH%lVPvqr15Eq!m+_PxR`H z^~v&Y2EAs_P@L%HepWo22;GJN&bl>{nY4n817XBMV3Jn=;Uby19~1)@_q9+mTH)!c zUmYZ~!3QJK3OZkJ?FTT9JopqHA{u|&*CVmIti_6ts_s1e(fljCP49OYe{FP_x0 z{AyZ~+?*ld``XAl>yK5NgaFQa_=cXw+@pR5r6zZ@bo;V&n2G;crIKkU0-HuNOXQD9 zSL{278E%bb1W+Ov-grhzhSdFdnjRD(lXFFsB?qNRRvQj+Axn95>8^$Di63DFS{Fu7 zpW%%o#rGth6HAB%OSNfZ&H^vb0waQEx?lY&cC%zDH(h}+Lz5swi4n|sFb0sDT5}pJ zqW|tyvW5wX;e9#`R&dww6NHNZ{l@GAe~zjQ-Kc+sH|e3Y@i%xFY@9ie)<6_b z1+Is9+BQUBf2a=DN|XKikcbkHsdo2%$S&63rK?4pDc0uHdL7(BlcNKHs_{G+bo|>p zi5sW@qe5Okfmb+ zHfv_KKc*AKOiES5q`gMBj|U&xRUgGYV|gfi6X5R;zY1f@z}a50z8~FA03r6&W4ME_ z6;O^`)_Ei?(ZPU-xXk&lE`>gCNIYEp_bRYUg@NOxb2J}N+FywJg|l2%)N{}0R(m?@ zOh%MSTSTY;ClriVM_q;5zR%S2n&m&NQCOn_XSkJx7tJ^GX z(W&FpazV=iJ&PLITxkGKM-+eq2|GDX4n*e5TYy6{2nPZsA9dAjz6%f~Dc^(#6cTfbL{Ysz;!c zY71+2s>i1xb;&ka_BqTizEKk4Is@L9_zL+XT;poUUF)*Md8|dix z{jI1d`6}=w^a1cC9{8duVgoQglknoIZ3bG_kPz`#GM?2hn1`{^F6Xlyg73tAQ;Hu; zvd_V@YwP=2+d308qH4$b{QzN{XkL+B?vnr0WhnL@yP*YSM2V<6bU|be_$DO>YDxMaMLnsLh$12muZP* zaoqGCF9aG?h!W14E-Dn5(@j<`BKZ-wJ*e4-5*L&;={RlzYuFkA+5mMX9+}`?5SS-Z z6BiRBU3Djb)VCB!fh9d%b(196c5_As%{T!FDVaYM zyMomvdrp@k)lfIj!npTnyx`1j!E%se$nKkQ*W1964mB6slT|!xq(YMrL`O_3tzz!? z$HO~ldg&YObIzI7kMtCcKMB0C74Qrn$uLAdg@ll`#0s()7BFB9;7EwdR;U8mdNk^y zM2IVLKanEfht&3PT!mDLxPNhERpiO%cR_-+SBEM=sv2n^shbav1{Wl?Mju~UoBo`( zv$K~|n*ulsY`(c8Z&a8L;AGW+i*#`@CLEDR*Txzcysh5L;L8V2NG`J`rs`yN7tlNS zt}fdmRdse<09j*p4#ayv;J2uSg9kp6wV#2{*s2=$g#|YS%F}?_mDN|KC22%^duH$A z5dAhz-RPy@Lt#4@Ip`Khx9$wP;B8EGHy`-wxC!I|CrcY^0vg#?dBE8L*0FT0&Dq8A zFd83d#x95{y>h>&9I2#Ga@DHT7jKRQ8fNsnLDp=$VukI;dvR(;7&+_l$aytduB`0r zaT#FgMZBa7S^FPJN$7$Wva_=fithr>D;;1!3^t?1M+O3Y6GH(ipp@6id{0UK>PK?5ff^C z02E3lG=*JPXO?zsfh`Rn%v9!#3c=SlaBSi*Fo`&DUHHaQQGU-Uv*(HB`DafZtNMv9>{D=O0!t<;Js}G? z6z0{8;*XE|yIN`UlDNk_tk3-&&u;yq2Flw*w7GJz_o04>JA0wX{@1;eb6Ky^;<#@| z^m$#6Fy_`X{PCwbNeUsEJa>J*znP&*V%mT1o^&Ssnm>GJx_4ONL=+uQD*F_blbBl^ zDptDD3sJ!K$5kb#6VHQJ)h#_JVn--{7N6kw*?mE6O6;6?R0@8J_qDP3&2RmXt&13O ztRv6_#28l|e0K2aaQb!P$nKM_(`rw!&lS^GJa^yr{Q4-s&^{VoAoGj6iA?6*#q6id zCLkhA4F3FM0QmQX_;kimuL7SLlBX@l1;^(DDwwxu1>k`REo*t`RND(#z_yB)H?ZU( zXQ9tg*GXgY~fpgGJJ6wd|^7+0>nV% z<_#z%DS7Dl{alxvj%m}u+3>q6O;p;hjNaGdm0S}a+Oy@Sy-z=0ZdGQ0>h0GB!8*9X z#^s!-Wbm+wlDB}Rn)?z%+{K4yF>&w97oxEFv{6|+=j0qf14PTkKIe7z5h z<6Wd=^SZM*0S7iO1~m2F+E!gJ{d;K~XST{FoLWYpW#2PWhPC`g@8DG&aAt~OzCF3N zy)E?W;`~q0`HjSmlQp?Z&9am*IQR)42bR+^d}Rq(3-5xL5;uDvcpg{irG_so*k*Ac z*RE{peF#n&R1oL}RuRk+&Eh0T$nsI9(>W?2%mv0DmZHcL4HX%P68`MJa$y0frY3r!n=KrsNd9JE=Z?O+hQvARGAJ?n^o zO)aP+mft^a_yr#&Q6GJL-VW`6nd6+rj(|N(y6$U)kd{2sxFcC^b66+TFnJL@^3z{G z^mqYX6S<@7l9eZ}(Jcg)RP&L?cbC zcxNZ0ROSWy1^YNsisL{JYK~|oYuh~9V2%s}-hxpZ6(My4pFxgQYL4tQQLU(M@)1kK z@H?R2N#fN5>b{uESZA=pA8rDYKIX-jpEcS&{!s7!1fm>a7-t!1)dmLlzHirSTg0jm z@oZs#{$i+`roqLAyNf@_N%&~<%oVnaa3Mp!*|J=CN4fn(g<-c|1?=VZiRNvkg@!QAW_|!g^RvuCcU<}kILr2qT#foo(6oYDk zm+Rst!+_%^;+UT=T**sN78=gM)(7_$fQEh@nm95sQW7OvQrvPsG2tnY%~{?l!`k9i zRJZP&n#iB^@>YJf5uK3*^h_Jj4Vk;<054)t!B4+HN+oEtZiD@LiR3Gpa|vbBU>pj~rUt`k#NF6n8!@+vv$=fvFO z!F15GdG*YE0K7GgvX#g5BB3P(tY1;&=kc0phVQO?#p0}DnQ1Hcd<|!%xNk?{$%_fC zUs9MK@)tVpk%=r6xq#=Znx5AF@H7e_@(H*o0bBwik5Jpd=Um|W;&4{q@vJ**=lJV0 z4p^Ll;^$}}&hk~>K@*`(Pys+w;1XI7O@|Ff}=UI0?Qsges) z0Tw3>SR7(t`R!mHXw*hxsX?pX@k(UQjkxZ;equ>G+I8yk5_+=~oQHDuNT!92e z?Vr7A_K8^WXIgCxAXA-yzU?;rkIsRc`y!#1$v@yLRQimdv(DMn|X;# z$?o|)8QMnKf|eUG{t0l7z-wJ;8r`ziH?PhDV+f*|p;fmn#IbEPD*i8~zB(?-pbK|{ z?iB$gbty?vx)lKzk&s+^3F(rS+!c@(0R@#3Dd|O8Y84R?1*AlB6#?mzt~=}ZefQo! z_J<4Z%$b=p=bU+-XWm^h!|XshkfpL4uSdZ)6?r3z*RXUe1^K~en)Bn4Gv1~J#43jMPUu4} zpvL1_a1qlg^&`z`ttE*;PuxAF4Px2zeA@YQ>8}mB&prUIIQ7_cJpbI%^Zdik$VsJi z>rZWSIgQl!D9!v7&VZ^piMh)i$2uf80*CO`3%3z%Y3kU`%1@G-E2h)3j+wxHk zg8|n%1)qRjz3TauFHnFF6|6gGL3c!&8ZtfRZ7ERFx)R~p{NrP3eX~Kkrb$HnA@}-t zxaS(h!)^b`d`XwJr~xXq?cpu3k+gs@Et&LSa^_)_8Tu_XbfM6&4Vuv7G0v_+rEPwf zb)fb!B9=T)F}%Lz2!ymxD`^44pPRHR%P)H7xWYAP)6l>w;yD19A)zoi*xniZ8!hQs zS5OdV(@n@r!NTooADV*cR(K!%V0M8bAUVyjV?pXb?{E1{6vX6#W$Nb!#;xx$NGVSM zy!Dd;f=dYfGn%Y35qq5EM>Ij4=WOVqBzi3rjPDm9yfTXm-oD8>^Lg-E+^CzNtY53k zvxhdI134luI9}^s&~d(29RS`xf`!sff3f}YljY1Rj$J^MT-Op;6(Q4OV_I1p2$J2z z-1Ey?ks17VNN@|-z_UN;FlW~7T2Ai)g7{NEDlBuNf4fJn8^3pB&G@G*K-C!}ZbgH5 z?+zfL)rlvI<$zVJ1dDeBdh@l$afg2e@=;^zdpRSc_hh~4dTh#kuHYxZL9R(;eVeT_ zBICgKiBk(-2|KCQ*aR#IlEjka<3zCSaKJj(hJr47;U)s*IDeIk-|82Eb9W@_8k@c7 zc-HxJ(*fkREXH!?)9fHt=Zk(YD-V!GBX6v4x9dGqXny@&XA1`>rN0FsL9J2>nahu* zoF}a~JP;A6Y+ck)AGtV8mnS9kolr=}yb;ly==uJ|wOqd2=arcn->su~WSa6swEv8~ zUfjNbpMlt~mK6wcnig_hO|&`v3-GH9S-Mb;`M}fN#!A zDcA)#j18oXO4O|PYXUACq~@w~LRtnTv9CxJ6$xs4Cr3%)&bB4pVIDV*T#>VE0s`ZP2YYZg55i2tmUl*Vx_el2q%Fh}naXANnj4 zY1dL8V-f(~V@1#jx}pIZPks0VB&EoW@0=e+Ia5MyEAxcoMZGWKS@P`YF^p>8NBaJw zYGG#wfrtL17M*2MK#8sJ+X$wjs1unf8?GQoWAF4SXvx5Zl={~rpBA(vq1q%EN0Zx% zqf}@(-3o50$()#lu|8wt8PDjK_XE%h;L5Zjo$Pc5UM8a2))g@{w#Su$*j=nC*6R`Y zMK_|G&C;f#R*CBNh4MNIiJX4=HJ?*fk~X4HFg}y*tWS-dScJHPm`lGVpN%s-;Od#X z_Ck0;Jbe8oMw*DhWbDn4f~pL3BNRAmUFdyQg5!_CZ7oIKm_tr_(ZsIH6yjS_kB#oG zQjAcnChmSGKOwh_8vb_=;7N_oDWr~K9qAqLXr=+$3(p}BAO|!RRCmJ&e1er(czEZ6 zr8^386S6}1_)lU4__5emFt3NoUb8vqLu$Nq-VIyvUSv6Z`yP=2<8=YfIa6|>{PWP^WPsyk!sG$Li~KUeUvthZc}9gJ?cVhlWS52o~eX|$Da{dmHO)0XSi zT~}ofKI3Q6gn8;owJOL*NEg%wT2F3$Ree@u@Qxyd+*AEL5a2kx4NAjzl=9PZL35mN zOP($thtOp%;;+9r!E6kWu?^^dPQ7sb6$#BS8aG!cgZKtV%w4#RW1sE{^CVXYcGn9U zLzU{c^OiU4e0rk<>!&#S)c>UT_RSRr!g$o@RUPWVP)7LFC%++X$0(ogLBr4UskP=N z_leQn#<&@dDuu1fnqbCD4jNJ0gaG(rh&zugU#jqEDA5p9X9+Xy%&L5RqwI8g`=AK{ zYI44qB2yaWY3>C3S_VkcL#VxYN>WzMfC_xXzllE z51gzL3qb0o$YdxPD0?Y2p-wPLrV)pbuS`#Cxc~(A-1hU2Qu#y|soT-g07jA|E=BQe z@)Q+0Dmxef-@f=X#LmNtSfjk73R{L3;JUCkfl46l*SH>po9yH|TJjHePiK7>^iiV6 z29F?Hg}$I#^7DsGlZ#6z@w@pk*jt`K1|sMhU#x{ZERg^JuK@MZ^~&!v>nS9B7m}L^ zu&QzrR@LnO#u45>Gcn5X%9I%b(=Y+4#f+-yFs`TMV_?=*&!3B z)ZI0DToC^CqR|mIqbR@SvwiY)K79R8AuTSTvrwIwRxipAuq?{oc@L&pRg$YNq6aMB zi-p%>UAsIL@`+6ygPwZ;;k6_W%3%~3PB)_rx~khkqOzO+QQ3{b6aS;KXD6JV22D5< z_4VLdpMS`2#Qd0_Id+NK7Cg2x4-QZ7cqS6F#X)io_z?$?yHo_Z5e@`gHkPH=2(o{P;*yOir?OiXi9j+-IWfLg>e)vI$h@a*rnox z9XHMUG9stL6Eq-yO>8{^NIm-?s^lMw7?l>T>oq(u6N|a>59kw)0(^IBEue3uV&%Ey z5NPT~3ZP`7#S@rV^h(16Sml%k?}YH-Rq|@wEJ8y^Ke)a2c-GNh1uafDygf*nBNfvh zZP2;JD&dH|B9r=kaz_#XtaO2cOWZ9#^k1`pzz^PbV><=4{ zbBvCc59!k=>_M}ZL|(8P1mTot#92>m!7u0ls%M}zMvhhtPY%)vbyv!}IOa%5JDv#S z0r#5h35JC`AN~zpn^lgkk*-nlXHLcuL^?cu&~Nf02Z8_7puXqhUaGVu-Q zj5jIOWN2upT&6tq37_#|=E9}xuLxhs-gN8+*$5bASq;vN$^$8m%HM3X1xF*hT!QyT z+iy@B@zFPQQS(omLJy%c*mhXEpkK75Ypk##2)dk@GO)emFurOTqg}K^*RGS&o9VdQ zsR!nSCbTseqg7>}K*cu0ao99{Bg%}4Kjb1%?Sf@{6ZK|onyDzVl6qMO9pPb0MBFfH z0?R!Q-#t$apk1&*Mrn?pRSO1B{=KmNV|HTp4yFg%wk#fb6aWA}l=EYgH>B~?EZAbW zhz-E(7aP@X*hngBIZ}%4+`c#ErAd)z=W(9_CUcJHrcAa>p19~AN(YqmZ3>-hXQrJ3 zs%8YC)A6d>XR|>eV2~%qU!K3tp%$2XFUqz4@`ucXWNnR+k(?vppTZl#QQI4Ha8t;1 z_C)_(m?gM7LB-+*5MM`knf~V;R_12m#j8^%>ZtcCl$IPkiUKFQ4^>vT{8h4Qn}ts3 zHWCc>|G5@R4+$DbCa}$%s_LQ|xgHI>8QcMcP9OZys(+AWsCzLuof)OMb+`e_3xzR! zZ!XoRtMm2h6mZ-?N(tHUDpeXZ^&TS6+{0|{#iQ zGo6^IFYE~D6*cVR{o{MNM^slN<)4ez{NCDBV9Sy3%X;24?=bhpH)p=>M%#7ii_?Ch z&$n(8iV2%99wGiPP_mEt_S#5RoHyX@%S(715(^lbqZYJ1g7VZ?4pN<88-qzWA zjmKIFoZd%n9aE~dEV#7nG!;XR z9fqZ@>cFkBRt0qpFpva=5pe%DQ~ANHMdoaIhHcY%aZ_jY;Ld1271xy)QoF~+y7i0@ z&p%z&HY=~b-r{7$g8#$V+L>vm?!4+2AhZV@Eu{~KtwI;cGPzA>(L<*8;2m3WveZRrBT56=pd0aI=O zHVQHiX^@+?H^ebxZJ@MgktwSpm*cu{`N7^1I$&9tvjek|UISw$SZ{(K4bYuVVQN)i zq|KAED^*k<9fzYr1!0VvGYQjW*bSC?$u_)ElI9FJyb^l%VY5^p&=?}vNs?fRe zWH^$rK$C-vQV*NB%1T7)UIrwpQ-TNW<();SR7t@kDg zG(Cq_d9yQL%}vZe#|))6u@}T+$`NvUF$uA;kK?kOK39+Y*S=K+-~t)F2MlQI*UT*J zDdktc+->eHN>D*9CUn@lc1-27cp?yp1=@dp7pl+b&S05jEXn!%XfK@YT5Dl7SlqMIQRMDtPQwv{C?OU(=TA}=_nyo@zuYfVN?9^Xe zYr>V&GyNGsxVCtEhfIfLkFv9w`_P}e)prKdJ2k`Jh{`NxgEuik!smT~`sc!2`5G?C zT2Vg_(M?x8=q|PHh4IhSK@b}^Q}_7Tn)g3wWHz&SfF0>iT^w}X=JNg55RuRkS2oN= z3Al4Wul9g0Pk!l3(2WD=W*Rp$K6$hS+P{afOz}EV(;jD9>=*okv?%ywMGWLKHi@{W~?HH-^mQ z{^pVH>KQaEhjiN~*;5UT%l0Z1@wh{K9rs10yD)yoh%l1{A_<7LsY(imbCky7JvwGIzCVIm_LoN<#HQdSpDZNRzv4# zr)aZ=b|h<~S(SPtgtB6Eu0Q$Wa&M}+U16>B;pmh8kqI|15p`r!KG^(J>#@j-scoeP zgdn0h#}Z(qy)q@atFC*ID)1^NgUK-uQ&sYru{ z9NqqjK%|7481u*>Zf=VZKD1gD&!C_ovhtB?6;RJ=J-?zz0a zEXBN6p61BL0>&Y<`StYWC)tRcSLuT}ot2(ny+@9F8522J*nzLXJ=^-k^`EKpF@{nu zr#Osbh-}x!TtbFGP#%yThw=n^AG#9X=e6}m(%i8&n+v5rW##BLfmp`Uvt2eMnPO_3 zuIXv7vl#hk!rYAgzFx|iZV49gKhzBvwl8k@tk2*UsH&~}Q%bnuz~ueBM+OMQ?!yB$ z9YTFlH4#ljV#3PvSKfm5Z)sfeQl@Fa&SJ*9_mP$UZCF7erGd2i7b?|EjpyXf^z2^Y zkc-&Cr7f{=Ar`cN%Z8OM!rKE?e2;qj7B6G9MYMSbSuP^FnGLi08Vzyd+2g%FCpsW> z$Uhx?~8Q5p2-S>but0(0M7VW3C4uJn%m2k&2_CgUfKiQ>{Co zf;c`|<*8EK`yO|e?)*eCAAL?da{FHK=_p5|m?Lt+_0cDSbz{ocDjit&$QQ6ppD?lvIeG%pH81`{ZPSA3S5?`QTRP=cIQkAcgCmw)QG9TrUDH? zM-WISYar(+Eg_ckM}OYJ(5RWUHn@AEKMCX$g9~025=D z?TQRrQeg9tsbJVYo_TPGMV2{*`*+@&o9^7*M|Ek2?i|!nSbC7!iXOy+#CZO*-9gha|aK~0`4WBmeOK9UB(BqxvsFAsv2 zsO=(*crIEq^?4ZacXIWDL$@f!BjW<~g9*Eg6*Zctu_#^PmmHD`!=%b&Kyl7CpVJd! zjz2O>hIrY^zfli&0B)EJ7YAQT6E3|jEOBe_n#R>%=4>9UNwk`(hOgBNZk7aH_u8^& z`>;#irWYvq`CmiLK*Pe&^3a%Rk{%tKWF2DuGiApSxP$`nZAFnDC$yY$s87Y)CjrTu zdVc1?x94DS=LzXTFRY4Caj^~+y>DqVabOeu%kApH)10D0#~uWE0uxb4G`9nHElrJY z<*SPKrdm0($-yBd>QgfmFTK9<3=>+9Oht^_?_cezYi=cfCMABZ_2xy=I>#X#_T}w7 z94V_rp`o;2J~}=t2ve#2H&t!E-0K%c^6IJavg`ETWVdczMy`CJzORjQs@9cG_8n&= ze6@eDp%t$`eV#2*9A00^urGpBg)YH|31^)$L4d284Dp>O1tY|Q#W0q!k!_Dl;%pG& z83X4-dJVRe@ZJTZLs(DW=*U~bQ(v{IJD@e@vn6fP;^!)0k4*T~)eD~f+XM&JwCPP z3-bAv4Tf)>Y2Cuz^;;%^8GR3)WX-$~GZ)nYs7~=0Q)HY4vA{jcfxuG*jr#>^htxY_ zgla-1;mmwJFSZKD1=WX$tPfvygid#I!9-r|JhCRA9dPyy%?P_(} z%Cz17d2rMy@HLg2&KCRYo$*SymvrM;um9~~ zhlcs;Z*EnJOV&hYOsU%7)*1bUEC()%&@F%ORtp>Xre8&ilSi@`W6K8esaph%`1HT^ zSKjUAV(r`=#%`j;KjwFl&G3TP9SxK^-?ZLsrRQdweQP*POc+srzFx6uI=Tf1t{SMCewsJ^UwHFTdQ8^n8*NhY-sq$6v) zgy8K?@g6CN@9m=XK#z(YQPbT5x0d)`@aTFw_+K~SZO3EsHy{gg=wuQLZ&mcUrb8v z)bgccxSsd0D1hOlWcjk0tb|b#M^=|n|2%u&_M(2Af04Ak>1+|q82t{ewZd|tdc`fz z=GUv5<-{7NA-VZGRYL!@A?J(ez95}t-;0u$%D4XW`B@ov^O^bHgOO`wnc~7)LFM+F z%*0|2jkfmuv^i;u@P2Pmnd6;c4Ly)S`O5C&67W*OHc(TjhH@P#51|@{q7x{;M<-NO z(6Y6gv5AN0iCebhzCxznyH~#X?JmI;b77s0sEVHDT^QZJ_O1Y)t7`k_Rng&sTo3AN zg`&y{`N;kK#x70DHxi5OcD6&OU*ErOziT_x9t{3H`~LMQRkRW4dZ38YP5Ca9H-kR> z35>b=^Gh2@hTQnz+_#jmyWz{9BS@9VK<;pK0!W0!uI#_`g!;Su^!GHh_-1b5WbvhW z5E1MkHsV-JmcMJb8f_PCQp0AzGnwRfn7<#ZPC& z;qW2hJjl}{jakyeYS3_oQ21?q=aD%arSi`C0Eo=knjtZ)9A5DW-gSTZvjDner+1?K zHE0x-;UetpRYI2Y-e;NHtvEtdC(sE5;1uyPS~f5bEDvDkxe2yn5d-f)ET6RglzBK^TLu-K|af<#!5fjo?8hE$`e zrM2s0p<};&07fZ+VfF5spl0^J4kDcT!Fwk6hac$)Zpy8`q`4ONr8(2MpBLo9-MSp8 zrD7}|n{tZRaeucP$BiP91%Iq_7 zX@I!X-OC43LV@;Xp8X0n=Be-I7**D@S1ZSVzrSEZDwmfRTukK09<@vs851JGehHcT z)^nVlAj55Bb;rko)*FVWO^<=O-})ol>yi3_D&lV-I_K@F9ZO+4#?ZBkWWiF^y&qtd zVOTK*mw@I+7f3PpZc{kRfeg`m`PN1aA&CG-uRiKaoe;!5HSsvSp{72dZ!(}ynF!*= zAX1TVj&>k0`N>u^ASAwig*S4&s zc31J(6z~&zR5^bD6Okn zTbY}h%BFi2Aod{=9KSJ@JT2zA%)>#fuZ zDvV6su5NjD_j(JX9KA@8Hzi(3#>c}o{!l5#>Yj(yNpZX9!lM_Ds~~0d8SW0|7idod z4wNGj&a>ZCQNf3NV< z5kB5@?2(jYNh|k`Z6OuV2GW?zY&wh;R3CFpKB)6)v)Jal1qO~-0I_TW3%iTgmP;BX zTo9mkUSGjvwvKpE_e?fG(2c_#rY&Q&y)W$*#=118rKcp9hBzL*FN$K=^2<3iP*?Y{ zi8wx9p+vsps4T*urX8tW&8F10OW zP_23~-tDv4)weuyY?;6I%YD^sh-6gdgC0e~}iaFh9qhu3l4qIQbPMg#-Yp&)_`UAdiyuR}U zAJp1-D&Bvem@J?$?-^m>005`y2)W=hVW-dTJ980}&W^LCw%C;Eg*WohTjn^7-wF;K z8@i7U&u7N6;f1mV?RonNmGPbvMaL8bhw_>=ggZ^oH^SCI+EW#&NsUah^cWp=zH#OF zS&NWqGam9#tuEdl8VWvCc;e5nz#|VQ`pfdi#A94xZ&NPVq_G@gz5A|LEx?x(4yNjM znb*&p{K87z0Gp?!Kz}!GP7M{*cjBCDq4tVIyQJZ#qoT zeqY!`VV*4wdl`@Tg)Awj4gH+FH5cF@n(h=B77Z$9b<5HYim;~v_?2rIj+X+H*~Np^ z^;}FHnvwi9HB+)FkBrIe#n23dVXYTZkV7E@WNIWgSqtH2Z?L>XuCx&QTM}(kD6`yJ z1E?2YK*vZ%%dYVUzfJ<1Cnr=g7}V9dym+XIKbrjtRuVtN-)EOU%%B%Izu;GH6=nR9 z4n8yHF@AoJGsrXoitGlxN#d@$vv2y$Gwxf6_T6^TkSL1!f=n%AwPpL+3p26ZxAb~H zAm#qck&*=a1KNUfPnCGv$%bd>Ve3yoB^h1pRaBLmf8x2h8TQbBG`tuOD*RY&9>sZf ze?oe%$K6q?`X==CFJpdzIl3DUII`1boT^0z+B+3~a6>-Ic{nbGF>6&I&y^K(@P}!r zznQQiq@fE}E3h>4p+6aW>lub0w(??K%g?EWKwlQgEN=tqMcPUwLW!O;;z)aK^0+@- zvG~i^z~f21_C)`QseI5sYD&9**S>Bm(0$K|z}kD7{O+J*u0LtQwPHZbdi~5uPXlKV zCr6$P#S-}h#a`SAjFhEYtFnC`B;Xg;;5bDbqgIZ=F9rp0R<08dNYUHuBER-4KF0ba z?As8adSvb*wp)Xg42X%r9SaU$4LqkgAWt8M5t)g8^BP=n#y=-+c7d=P zTg&|M;j|vC5<+3?!lw%jPcmkW9?Bm!uPF1LV`H?;z>D;M&pG^^7&osXZ;z09^DXjg z>a*A+BgJ9(u8yO4+$>GH{Sr!}%ueq;f=tRVQ&7l%A1kB#-S0VR<<`^L#d_6=^9q#)X} z0BcP{8;YI9+;so^=UeZA58I)$(YdW;N>7N)`^sBva)R&1?YUeVpd7Ca!3HbnI}1ON z>A`!huIZPLS9*-aWM)SsqkM9!M{jpiEF+g)-P!-FHSdi^qO7ur+L&xV8?=JzKz*mM zK<%2&1WzOPC|&Hrxw8l@UNY z@W+PX-&wQ*We06FTWfg)DIj$gDQ6m~db>Ul<#BLmX)CSVny za0?HdKQuG0E6MoeSKtUuz?i(9T`j@}^&S$MM2$|zRxx|uRLkv3ofQ9A8m*bNJz39n zxP39#sKPSkmx^|^9Tmt}$bpdhE`<*B42S#P(@tLth8=psT~uPAz3n`Zb~twjMTj&s ztr?$tZbYlWigG;s;n$a_5G>mEGTA8)VP1hkLw#UJRGY;CG&&4on~xF=;mmx69Pg(tZu&q8wD-!zFY?H)tx#`l@`PT-c(}q4JgMx|u4$w+d%WS}W-~M^-SZPo$U?=QJ_vG1 zE4HMduD3_2jSA2}_!=}ymi=U2S>RZxu@vuZI$v{Krkxx7bKlS=_dbpHeb~bwB1}Ur zL^Sq*fh@;y<=YJZ=+HDXt9sb@-~8S>Rw5_yy6E{8fn=2y_fNH5>0-@uMccWmfBIil zVIu$uraGANuQo8Alhby}0h#{MI|>Kr(YyWKIKJKcls#qj#?9k5A;wlgkE$@nvujFMHXcQTvtP5a6zMcHb&fQcxP!JTIAlG9NE zZKee2gHYPB<=T9tv0Z^JL%*PAfh7u4f|r2UApd=)C~yyqfg{_Y$A?wJl|Eauabo>O$1!PGwl^J0QwHG zAPcaQ;J!x$7!m5e^EFho%dG4on!Im&d?G+%-HWIMzLaYO-8GRYm3u>M#AR1D_@yCz zuoocOVhi5!#6~bB1+`O-Xb*cvZe`F>ng?WtZZIh{p$wSmvgkr_vm2?)_=W_TNbHqs z5!oklwCAXm=#EC@-vu*g3VYMBw}K#!Om$mCODa_GuJvzx`5YEZtfsUjmleSy@2&6% zLYJTSJ|aQesbK?&imxjfkEV_yp&Y@O6t%2-o`zTc z?xxI&p#{fmsux~zKyG4Yhr8@qUIE`$-`tA2LORemax1X*%#MqMVP4P(BUA4l;r2WT z#^_?TkTv_lSIWHsrnxiJ$xd}!?QoW#?$^x8iTM(TXD`|u4dv|V2rz@L@KW>2n#Iz?QVO5{J`l*eo@CL69e zO}=MwZ@%M94(*$w=9oH4dX>o{UOe+?us&z?rt$TNf>ITI5|vm~s)E-u4toj7Jh*>p z-K~5*lCn&6nDFB20;y-D>Va95@o@|6_1eOJn9uZ0%3nIYF%dUC$xjXzhYjaf{b!?9 z$K-19af4lLU)ykfXs5ed3gpvuPo#77Kctq$D6>4c5s7=qsv+mAd0?MJ`>M?z8!@mk zT-`K!_EUH<=$~ty&uxAhZ3gJkvNuvbD(Ni&WHP}XN;0V{H;CLsQH3@9S<?VkIO@Zf1ad8e6*|XQK3FnnUqBi9YB2jY}uvWK_Dc^_fHWaH}muw`CR^ z_K55wXWK#ydH#2cPuP#F4<|=H;X^X2f4|vwp89(l@!RwvvXl7%?x9P(gJaU)u2TKF zk$P@&ih|K_2jeh|-CY-l${H0ej_MsHJS7pX(y8fZ{nC5KCLtg94gD=)GJ9*%5nKlz z`g!t6382R6K5FmH4i2tt^&Rv!UHc?&nuuYm8*kr?960h2u=*W16$N23mQ&~Jn6B%B zMRX|ewvkWS_$>5qL~oZVYxGI|G)uvB-(C6^bPEwQa`5St@0{~%BQkyqqhU%-qBtdS z5HO236dRAJh-(ClQ!95Fw}R=n6&u!@@Nvj#-VE)huLOpU;}l_yTk7iV?KQ{#$vJ*v zTqT z=aINwT~!D4nze@Bo5XRUtW1HXW9ntEpDe`rYeEu~A`qX#_v2M9FMe@0F!c<|8rYs; zG#LD!7T^rP##O)TjShTIejXD(mH7`_LRCime2*kbIbAd|olu)MwP0tF@4o6;6&wD& z`(7)pjLv~Ih6{uAmXa}+DwmjV==?_T9Q{i)lCkEbUzVBRd0s+dLjKaOZO`=@w_H=_ zJN**yZElSw%^~gc0E$Z{yGc^^t@9R>KHbrGvuV0#l$Nhh^R1D=jU?Rpck&8W)qZ$3 z(A$%P?1}to(LXKI{=BHcJYtVKQ$aY@Lww2M@aXfvCvEZFI6EQ!Q^q6(?mlQhD zy-sESnnBHxPV~7j;tgU%uUBCtKwLaBG4l6w|MUWCgD;s3PVWW3$-n<|Pg);ACnY6f zkm@0<&ASUr!=^LHL{PhDfAm~?m%0}It30_=SS`CacE0e?s=iqd5H}z)=TO#@Pf?1y zH;B=NMiPz&mlj)OLlC{v?ru;|=r|*1sEpqd=)<~&nl4bN$^>&t^LHU4) zJ7(-ZTHZ*#k@`9*4iUy}RhK@F?ma2PBI1Kj&aEa7!E*fvmTP{V#Flo($D($$%Y`ug zrrJ4=(1$DmDFIIN6K`7CG0s3JumMA)D9t9`YDNUgm#Dwdyz5ecSRz|;X@HS*Iu>@k zL_5L{bTz!2QI*nlp)vNI>|dX|5~P@*0I&Y$+8wGVOOf+We;4~#@SeNE9{^j=t1oe5 zSn;^`X@?IFd9@)t-d7}e%J>wg&35I-NhHxJnm+MXb(kXN?%RL68B9lVCHYBQlC?+& z4VM(87?ONRsmY^Jb%1VyC<=mMlMu6^ zuo;GDLchxinl3>Y;iorcH{D4=17#u);Yg8*l;(2Iu*)B-B2R}>{*OUIMFh$R6kh#Z zzV?7c>DkME(trQ>Z^Y2_}{cGK36Oc5h@2`V!ps!D3 zj&LYZoG68{B9c4FKk+n50)Bx=Q88V5r2`v9W@ne#lTNq4yUekY*h8EtUWxM%J`IVn zH6)4pfGjR)l_Lxv)>D#+!y%UIuzHmyjM`}`57NhbYYU2_V~M>;HFS0}Sy|Lgxl_AWLB zU8j%z3BK}xSD_df`Hz?%I5o0$Q5|@db`A84*nNTH#k=NdS08b9L2TfPmy}3JOx`K* z?YF|$6Bm$3v)AC1%{0nM1rD1tz$ZuU!$;5c^xT64JoRPu<}vyEQsW|N{M|HO0!?9I z7&?8jmfY3ef7O8o5Vdgu>6k@rQw4O-{=U``Ano|d%E=hWb~=h;DrvLg{cDfeUz@~| z=EW)}ilkY(|0hGsQx9{0zIzyG8NKmnAQJh2QtBQ>EB&g_mgZlI&bmlI|Ge zGv?$VMuR5@tpXS?3=BUZk5go$g3tb2f&z_b+BZ2Q-Jk;wpaEyr0|wk7-Ux}p{3OtB z7?#bf+1HZ`z(L=^+s!wr4`08^PCvv|q`8YgauRR0bhP01u11$mhnxxO+_#uup@QVX zWnmfrS&>%HE~&NM>k5GefEh#u>_ztkj0gJJ17op6yi@Vm?>I4*#mak{Wyketj&N)l zmX_rTlpQw-)^^vPYl00PV?V4uFQiV|awWGDkv++G+!k1oD7VkANhJB3jt22@F&&>n zA;M$bRmCE4OG|SASyy=nb~}y^_4OXH`GL#C3%q%V zUcL_+$R&98^3_O(3$B0sDXs@QU3XY9Qf&5ModYS)+WlU1A*4%YA#uUiO zaP~Z!O?)+wH@g)VL>tg4@|2Zmz|}br7fq-KxUr-A9O3IVX$b=$8UvOfs4?o9+iIA| zsbC4fgdn<+z_bkj{Kyf_U!FzNS=s>YozU>{=UeR7CA4f2^zNyzek6ibEwl>L1q*D2 zU&yzB=DhXZa3y*W_rM>#Rqwk!>QqqwoECbIXNmp;hd)_@b|6F#;spZ|FbIOEBxP&? zCUh+4ZfVeU1y*PbRI<936C-r*mcsp6%EGPxO!8G=hb!h(YV|Gix)9!za=Z<;v$Dtl z+?P3v0b9S{K>@z0qph5l&CA$iKtgtUn%V2Z6lZc!16d_mFGPmS#Gb#YZ<``j_Zy4W zoZCxSBUK~CIjRZrEVAX5EG7~Jg8I${Re~>%&DG!O9hf0asC=2g?`DV`4@coJYzV?; zLS|pJ-4*oYsni$1nRe*I4^uYS$y9~jyCRa(AuZ89cGYSV%+dA zin3l1MJmry5bqHE-PL~wT;p6Kgz5ihw%^ENZAbt1T_GG22Kg?NeJ80RQnC!0H(4NS zFva?&CderJUZBpWKn)Fv0RMCe&6NM^Jbr8z;H93K9w3;_;gmw;+%-1 z1a;&4{pJve-1)M-W+PI7T8OxM=k^7?c(z*e2VW)b(*}_RRmpOzchai!u~~JDWxKBE zlu++~W-0PKEbIwrH+iid{DIGXXe`tzOc0jBM|E@G{@vl<){YuiZwn@Ww zv7SYI{P?AQ+}~#k7%6XNna(f?8Dl>R{epGyiNCyYbYlB1N)ogp=jtwXr;>z*DBXKS zxXQIx3p?>94ac1tnXH%>YWxE;Cs8c2u?kDg@ae~nb;7_2Q^J?IiR8p$G~%jWD!&-e z0HtaG#o8=k61|uqw^MO{ANZIQ;sKRLk1&I8Y*VnTge?gVe{0-;#dGdLO0 zH%nF{-1;yYHyB_h!lW2*UzC~m)28o-F|AS2Ob^^x&Quc!Tn=$S)agl>C*%;Wm3OsJ z$+5Lh^rM{nkNB{XK;?}`4oiX#%)!f3T*T$l zWf7Nk1uj=T;j5F(NkNRTbX%>(-S?UomAL`R1=!l^_{V-P$@jQS#Qs{$*GJ;-v@{Em zGUI+{+NUq6G2)N_3f-k7Ql{rq{U>uI10kM6axR?5-jIOte_-n+RsBts-E3`zCFsl%-JUp98$4o6TY2X+)fTj8 zRDPp^cLx8BcU;(DKy&c))IzWFx^1fXfvn8%&+`~nQO5C`;Nh(Dqkr$DSU1SAkJ2=< zjB+piseqoyP4Ij>dl!*X*>BYExAN!c@OtxhZ^5>5A*eykUcHgJN;yi_SnU07u``pW z@xqmZCtgCO=1r+&gXwqn=2@5YE$FkTL$ke4p%+^V1b&M^dKU|R$N-Z};?rxd0CFXh zML?_LmS~r`+pt;kRyKmd$ipOf@|QDFk5G)Ro1w@5VpoK(U4BIIw+Ub?VL!v1&X0Mv z*6@n~a`sWQClV zfn{7;enw;YdL8A3=A?J0oOt}In}_Wt{8Sbk*eadlV*Ps4@P$=8l8j2kqX6p>&sI}V zL-HLSaiMo3Vbd(83=?5z+~(~zA*xNDL~)C+Zk0_(ABJ-+>5me^AKxfe_{~GxLPIt- z5o!yaN3|RGG zm9hDFJBmmgEtZ;)b1hqqPuaaRLFW4}&!I{XOy26QI<($&5q^zg)goLF-T!L}j5)rY zDk!(BWR1Re|NR^`S$p*CK#MWgW(Lf-e6oaM&9!s@-+&+Cf@+X_%73o7{~ZhI4Hl9a*e`VteWO$TtxKv?b9($NN0_0Z zd##O^jqZBEy^{v6Qq8 z0ckY}8ImCP@{A9k|i{T4gy!&^W zYkblaM-c4Xz`MEZaV48>Pfwq=kN@A`M4%6=-y3`K>mk35nbjcsX08%6G7-;%eLzU1 z6O|8YoGdCNG=VoP3bh)138Jt7ph~RtQL(Qgcl~7PiVHz0r!EMBX$)Dc=n3}J?6;{R znLcRV^0JC;0ThyP4jJDe;x^m8RF-t_Tf>ISrsYo$b#uBpb#BfCJ`a5Wn*I`x5sgh! zP&zx>hCFA{jy%_VBc?9ap`^Kb;^}bj0r~|~?fQmntacd#8+mexDScCFZzBUtN%E=*P6HG7MyfxWVSnYFaxG} z5hRIXz>_ojt0$ls&=7ZeODZXQmKiTfm7+CGShJ23zIv*18EhDNRylI2Y5@y9xc58n z2nR0!C>78?u@@8kw`C?6=)zoP?aOd|skL#cHNR|XAgb$^lbaD%P0Mr!?H@G=MqCO= zq)YR85wfl58f$9xYKGJ)nd!UD6s zXf{_~1<@7oKPj)3?$lgc46Wi> zJF(yp%0LWpNje-Tq>_{24)<}83mL0**0QDq0E<f(djdJT=lmzFOx?=!!ZyPM2J`Ns(0 ztSI30MV1yZk2zWkkzSI$1(j<`>1I1liz2fh@bq|pTMl#ZyniHtuh}P7rx9MwLZYh= zp;M}=qD2b~y}rLCd}e1^9Cd4NqOP%J)~yKC+bAj0AW%-g^{YlV7|RU+f?E9?LGD+T zz~K^A*~G3)f!4_nIjp@eTD2ALg=v*j{zb z;YEd0_&lAy9j!urcfq2#Rlm>NT8NvWYpOlR3uHD6+4NKI$66hkfm|Ltwlx8e(}OC# z`Oq+N_e8zzj$B}PD0krPb>~w_Wnd+1f>}WVm-n4Ix022F`72AejyP>)zlKw}G>)U% zkm`4*ROU2FI*$T|44;+;Mg$-fXljjIJA8pXo&|rH@x}!6XVv1r=z*70f1Ob0V*v&Q zp#kIE1wM9HHeAdefXMeR1 zW;sq}42>N@(LogUiu_B&fAQI~t%WCwos8rmz4@00B{Nu$5b+FQ=6;l51LtOHNJzNx zW9Jokj0s9OUYH8W>vu!hH+J1A?$S_{;L0oPHuGjRO z8W=7C(*5VxI;kRXTXO!voIEFqg8RpBXMr$jKgL;D;{GW@`px+^}U8VLoJ036{uE9pJ?ZVGc&dY#z>*?IOO0 zU&EU$-5QM+GL_)?!#G)Yvr^l&!`^bf!NvF)aBs?JcLd+i?xaIOW>-^`g7rp$V2I!k z!<@bO_uxKn-C@C%D3P`YTGDBI;~;KeB}yg|O<>i1X+Cp5fMV7H-C14fedWBzxL z6f8bydTY5|>fQu(JatjdU;wtVkqO;=dC$8XQQzY#aL3nmXGrEp{>gB{ip)sEGr8Fk zX~*hNE@fKozTedlq85ny41#ho3SPI0<6fPgU&1-9F+V%=Rf(Bsc*;V}%C&?3bVUH~ zo*S38I^J_={-}hVjP7M9GAb_2hP00a56#Dc$oSDozq4|q!%f10ZSVUV@g?}O3a{s? z0Q2>@d&*F#men@(S}2>}tCrNJzsb$dz(tmV3%_JLT3*)xCx%6`I>!;boQ2Q+L%<)G zzwor+^#hSMO1!X4w^6Ch`?Zd^?zs`~?hyGv;SHMy`e1XfUDRM`*W~AZ=twI-r#5tB z)X$|daH)rS;PwbJRcW8=$?$vt1G5gnNX7TRxw<%j4Tu%#zY8vGJp7w{bk2n%oq9C} z_%LsRhtA!WH|!kmL{Fh7u5QG|xN^=oIS&biD%g`YW) z1^Q2>#N#L6{%)`gxS#`ib_M8g%Z~7EMy1(XqX3342OjSgmcJjswHt(-kWu>hmcln` zHCur-kGA=YSQj5oZ~Y&E-dD!aOnI)fZK?{^2I(93I5U<4E@U+4ev*o5S}M)};MAa$ zB9JfKTYwPYpy14-4*cl^cc{3or0Z+g=zf^>EPn2OHp3=nKkNEe{{tR@pi(i6s^jvD zx~v}8OA@1^z|gN>C@$doiTgko^!5rzQNsYuA{BoO$D$-e=HuFQ;xWRsX2&DPTu&H* zfs@ORrtbVUes`+Y>pgk=ER`Bhk;CckBBmW`xN1;e7kn&}v8M)j3^pJbqv%{Aosnbf zJWzfo!mY|j8fm-b^!K-aEl;ib*WWbD>*bFEkaE4fFtq_>w z`V5c|nvL+OdlRxQMCicdz_Wgwf#xV1iS1xEzSW4c<*GzMoAR|cKEYXZcMD<-nf7Q0 zZVx=^@CoVbe%}i&f+;@uDLQE5CaT&8^p61LzF?=pPC;d}`4A1x9yl6YcMXxUs$X|P z7vI2ZD3#A}-@Iv&xtkz1^dQ<2Q(w*>W{Gjbs8`TECoi$kXE(}(#8#PdEJt=W%m_h)~FqQrNHCqh{b*}t24mAfR7uowA z5iSN+f|iPRndY#0YgtXL^qtQ>DQLvv680U()8y;X7H`tLXgDbz>zy#`l!|?b+1FyX z(lxtyahRO~>Wscf!EQzoe$k1YB{@7LL+?yfdD=DU6y#Hgw72BT%X9mOXcHg9%C3Tx zuB`)y!MZ;G-^MrYLRR=mQWQM}@C`s!qz>ANQX((i>{^fWhcs9eBr4;tg zsW+gagD|}3>$(#0?Va8GuO#L)O)|$q)6_>ltma3_48m0?Yw@5!KR7uWgN!lyT+A%5 zWmM*@?E=(OMIFSR;DK##DtmZNW9i%7H5ym180=yl-OBt87k)ojvbZtIpEIBMG4Ilp z9JxOK0B_3D>-X;mP*hRZOu9&eR(C%?GnZRVoJmCvcGXrkh9RU-Beb?t zh0mXejyWWAN?UO!eBHWlXSWccf2zKLC!b<{1(SDPe6&7#;qD7RrW}ffCtpy8e-WJh)K~_ z&bZ5+XT|x&Kn6dLet2C_*OWI`%yqPAw_{Np@Jb1kKPwagePz+A^F&*-n-HAgO{Q<& z3^yJ)wg{xGh_^g&ZU&S4i{Zka3-K{IAtk0U)Mppu=k~7AjN{e~wxZWeYaXi&C59HX zm*81&j}y@c;1)GsUW_KZ!5x&=N-Me|k7Wt{=6dI}t4Y5z%LUG2!N{i9SIt=hsEBfY zimthfqqOTq2f4{np^}GnP`}%S{1S0oh8Bs>&1mPGqlA%n@d8F{FJ!x;b3J@v)bOS6 zcx%){&65N3E0qkjA6{0Knw{4$0k1{^NQLYgjgge=Yv>Q#@0Ed}Etlirj0p&&yBl!F z2$u*MQM$v`>VnSq{n$`B=x}-0i0U93%JF5ffLFEI$*HGlw?Jh({!gtMc~rQSc&CH& zd%i(y1+y>oSD;~16SI@fU(`sgWsL!I%-EwxEyUimB6Ripfqs|}@2^=Q~$O) zkc|VMh0o0*8_V;Z76eFAdBIttNd9YLFg}Q8{hb3>-sDzda~LQeYR)CBcg9JM|979; zPg5a5kpRW{KQh{{qjBy@=a3!W{%EZw$j``*2kluh*=l%g7*9m{U;Ck{v=}fTZ!dTk z>dJ)i-rZ5JpCyIVv;R#nHSN}u1l|cXj?W&dMlU-TV0^l<6i!MtJNLH}jZoX0v-S znT}DkP4*s6@!_ei1%=f{bGO4!dvlO^qv`UJBxhdd`JBG?nHtD2ztdLi6v-aEyPn+K z>?6X)ul@6C?<@{~yL){G{W~z9f#!;_AEJYtUj42SAJTW=RF0PWzIuAK`UzfJyUiWY zhaPGdoQhkU=F&p)j&VYgO17~(#cIDJ#mbT#Lpw>&LoHU?!HzbLO3QbzWKQr)`CNL+V0Gn;;3pUT(qxr zuV0fDLlDw8*GY4<>sg!bEcdr#Q>@CTu1@_=E&#s=#F7dfW~9w5mC$g8$e~@;`(U-Y zhxND9=rGVk9p)lZ-OtIKd<7p5;18?A{3o_PA*->AhifU@EJgiQ8=Gjicj}{=K9h8S z#ItYyZIs0i4+tOFo;lmbQkMH}-Pb5wyOBJpps1l7H3z7Nj^ZB+J5JF5l% zc|c22ChEeRlzjS%fDA~cNhNjq4|zN>D88-+F{U`=FY|?{!5sbsxqu2=r;axkf-pc6 zK^eA`c%+KT303k@TVM2aCLxkx6XQPL_w8kZ*ZL%1JTjr2fCQfN`rDapjACAukrhN0WZ;mhmuX{jOs}1j!irB;3D`4Cgj<;pJBh)y-HdgH3RqxtPJ{=EkqB}0P)G7`C8=_ z-?JMv5uU{ovb+zkm1B{)^>!rNc z$fUVGyoP<|GN|_IL&cQ8f-vSPZ$?((s@XG<^YR~*v-?0ba>`d^%5g>Zh({R({8jG< z1ZW+GD7~wX!WwhAyyk$+m=+Gy>Vj+DalQV&1G3e*!3(}m14pl0Db+9HMrA`cU#uT( ziV@cF)hx!Bf0%DYx=C~bS+sdD}KX3vs<$f-l)n={MYhRbJ+MI)W_sgnSytq z_fjdCHZ!}5Ro++EyfW8o(u1}a=Yy?Mce((9nhY#)-7KxDE$Y+!MZ9&?r)}D(-0FJh zmKqQ^mD{J@AZCBJTolx7dRcWVjN|hx!llGRLMeOihX9i@n{9nkxcExMmn7z|xLWE= z7LJH7{m#LW7A4zrW6qaOn$%fUIh6fm-WoIhHaB;}OC~F;P$lx#s~c-_ja}2Q6$!>) zkr-r@S3V>Q;*a@3R_h%PKv^zcUGU_&_@^q_rlv(L#4tAWc6oSnhI%p+Vfj{0DFw6l{L`By zs?pUy$gD8V`GRtti`yNyo1f?mH5CdkujbTvzQ>glzl#@oQo(b;>Gq$hLQn^4O3b9F*J6w*gEX`= zZZztb4d9yaT;VapWQ3<~vmckcxf7_B)<-zR6LfY&2#a2HmqZwz#L{qVez=3~fVnu* z*>flkPGK_G6>P|w2_Lws7wRhu{qagEAk{(dD;P>1zri1Pa8cO#ybM#a2*bBfrumZy zr6#qye1FTgzFrg#WZuX2g&$akfBeKDfB_bH|0ZxVvFp^sOTRx~-#cy1oy$<}CeksX zI+R%7M$!!)xaRCC}r=8=gn^F&l7jXpUAR?ly6G-PpCJKOCN3r(eDce zif!!vHBxo8b^Qvo)-T!Q6^jHJj+aqO|a*bH%Kt=ISnExTZHtXc|u? zDFTM&ub{-;>>K&v2!1)Ja-5mKYBTmJSb&JzW!3Nc`|N6*flTvAm)vzmd6w{{v|GwT zol1h|5_}yFGU$7RH(VB$z8jQYnEI}pSuLgWpGiR?Z4Ry+4vG2oVKft8*(eEX9o zbV78)seYi>5H~paV832-t*w$?gMCn$=cvprpAvje1?k zWaE*h%it@kQHvK=7l4n>1dmX~%Un4pKH7}yT#0RwKawt&I9>E0FqDmC4G4t(&UykB z;sF?ULmCBlRm(?HqfL%{-}2^MRxZ0Mks}1TfeWp=h6FQ!AC8L;bhL8ooGd&63agL8 zYkmZ^9M0fVe%UydZ?CAUHTgZEXn^kvDmO~@->}A82Yo-8Bs2ly0&TZ<2UQ;<%wZF- zt)v#p-z-tWmYtYp34)T2a z%^4kxCa^|*MN<0m&^WGNWbQBRopbQ*d*jbRnR5@zqNM?ch^v4)i-E}A>;G^#1iy{- zJ@wBd$mG#dSzMGjx_l)Nk9E1f234m?{4)CDfW2h<+=foN zw{P=KwA8~C&t_l|>mQnN`0XqXjC2G2+s-S1jH3H+7hi^PBs7nF`^jt)` zBM#`h2g%<3Jy-n4d?)#PObxhq^F58LC#f2;T1z#R?1T%c)EL7QnrxSqXh|!eI~#Z^ zg{$(uIlekd&>OP%DKD~~T$))b3H0>#j0}Pp##EpCWTvcZv-{9XKQb-*!k3vEAj%~a z?ECT97*!#)`l_vzXr#sBZSCm;-j>EAT5$bmQ@OnYA;~;W$)N9LehBoVRH`UX?F@0f zj#K8O@osipD8pVakh~u1?vo7Oi;^M?jw9`-4%)By56!&>oiSe}bmC3$3#-+#kTS?9 zA9+S(ngh!3sa~O(a6+kyof)hsL$iVO-oh);}_RG;_=trmusN-(fB< z$Z7%D^vmv?Q?)}#OmSOZa+WcoCkCIb+CT*U`~`u`3(@}>DLf)uz;;h z_=I=%#(X-t7m(Hh?wA#-Gb%q$mRp(_pyCJWb><@TODlHAs0pt;ibGv)-d%Ts!s^-1 z6F`!38;ryY2~RCh=g++yNkm<$%2E4NU>8(Ybs3XI)A-{g!`oo~p=!zgEeX*QmW>tu zt-6{a-#aF+wyqHZLJQtv~V*nH+tm9lm z0_e|Rg3pvGDbKhhYp%otpYI{CB7B(j-C$+A}`a$Xolu5T(y4V)qg*biE3Z&hp{AxLWpXxGh`L{1=~(FT@MJVrE)Kbs&8 zMuv$(%mtDa_fFvoL>|VzKPAgVnTC)w0Ihpi=)6;(58fFo8yA&K%jUy0Lu;U_FeD*Z z+x3YM*%`f*p$>}j7BSRW1v>h;n-yM{h6RdP(TB?(<{s)0oV7YYIAulVDaTnsf!$SO zQd9qOYmQs8@Ry@brtgx_>MpyBjNdy@Z~ZUs3#W%UnV-)ZkR>%+0``e?usL;r7&rrZ z3JG40!fRovO)1_ieyhM}<#CG?o@s#k(1QZtaDBK3OLWWtFL^>SkIofd0)$e7UM7P* zP(e^tS%Zn6wxJKOTJ+K&XMKU4!xb_LB;W_|c_3fa2>S>)2Z`sjEr#p(hq%#FPI*-C zS?TxE&NPZKl0qKoK!{qhQh&Xix{R7G-$WmN7F~NF3;qD|85f z49R110k?^V?tlvEXSMV1-vxBm*6F%d*?MG}0qYotW*=WQoNtkeqh`9&Wi3gF?GA4_^cWTRQ`5L^UKv&u^>pal3Rcy^Bk$PExuqs0c6_iKf)^LfJP5jARNs41{%Q@IZ(<7 zieWrFKO}~fq5*(nEMUprG6Hpc?z9loUW4T6DqtCzxrLE98ziSPY)B9-JyEThe`nqV zEcc}hAdL5eiVz%*z_KzDZi)&c`^tk;HvUdu*l{maH#%1mLo5ScBzi@4a%Y0BA;)08+s z+PMPFruBg}Q1A07*GcwQf0+lne>EMFN3|u&u3!sSj_!SdI`2{~3~F5aPCURBktN%v z#qlqSCa8D;>kFj^J8YD^1Vx`ON7W53Ck-xuB9{IjaN@XVRiGD4mh1|k6?X7l5rTsB zYCuA3Jl=vr`)=~(^gs-7q>w2J7eTo;pmsJQ$P&G^c~MY^I&4r5XrQ}?T0rMKMu)Sx zF!_A|iqeyk&D$akC<}g0SkT6gg2f6bmWe|N$|ic zLoqi_rte_UD~xDbwelVfzJ~K?>)i&;6M-W85YA}} z?1lu`U7X|^=s#OK@iDaLCqK~q(08Edf(@Kk10*@uz|5`zansXctz|;XPVlo7MJxeX zRGmBp-gQKFXDo<7g`gKOq6}amwa5#}WBGX{oX+oja_pcpMWD#o#f6>){D;PBb%Xg9 z8S9r|2V5hZ1Tuz;ekMSI_+9X$ljE#qB}ow_wcX8uPhg`WF(4>rbxw^Hl)9r))35bb zN*=~z)d#pn7!YZ(;1wwothC?x^Se$w6$18kS@8nk_zt|_$2j$QTO1!4#W&#$J#;|> ziKIQrlCbYBr8#?nJ-`&)wPDZS-tLPFOi_Jb0s8>(NaCy>b)2i%Qfc`z=^f&;kW2p;e=&EI zJF-_6DkzMYr#cILvrzGK$Opk;2h^)on2FtsJjXzw8l7(d0!4QMbyT6v2Va*JhBJ_@ z^NP6Cm{R}`Y)JLKRw8IS;62K!-E!s}e*Knk*8wqD+q4FSc}*2ul7+1n?9@al;i5mX zMCl%CF(tA2f!Tc+j)Eb%*ls;6gO1URU7Y_ajtOsDjzE{Fm<}-Rp1Q(e{P}ib*vk0zji%+{1UX`L+s=qphtLK z+BAR)sy{An1t<5;_#7W#us-;s<^(8{PV^-~w>K+$>}|gp_7Ux++YbhmtomOAn1jKO zfp}4S7Rsf@bjl-PB9#xUYww#Vj#HtOXdR0;{ zkw;f$Md$SHc^kD53xwb+hW(3C-^3IT)LzdUTouH)@5ws1T_CiC8FJ>m)Bp*Wv4{xc0J8qA_Dg*$5yygvd zFf5VJC8Gx0nDDa~0CG!OwZcCJJtl1l%OC z3Z4UR(ve}2wxQQ;{jB!g>C3=%rH}8Es*}?=+V)`ZyG7eLd4TN)F9Mpqn4f0#N{~Vc z!OjV{6+&~@7Il?b0^Rm$c>rgn4I0(}M-F%+24HKx(|_qV_02V}5lr`1?o0%dxuoFi z0A*OBne;{^tzHOIQcru{wKhdrpg>v8bwy(J_s_+AcZ^~(@n)mS`F)4YogLuYtE9;? zsB?Ve6(rFFuppdD@zUGrvTuxX0m+w@ zbVOdOhDTaQ7C2e(zzQ+E8%<028hEMa5oF=)^bR=1z%0*vM0sUZnswH&j zadORv>h@{|qIkJwAw>td%D-J0-#j;A)qpQ$X)wKy4~V!kMOadA+>6(b_qZ)GrZfRm&;^`XP@FQskb#2(7E55zgd13dt)6 zSQgd*S&{`sN4vh3H?-@lK(Cw-*OhMDKJa&8Bpg9CsS_zbg9FE;)i;-lVCPT#whuk! zWD2Xq=5g~Pd2Q_Bo9gM|yTdCiEgJ~9w8Ci>GaKN?TPJ{EjMb$F$fG{R zB4Y9?!7J%mfmGE)IsSP$!X54WCqhtaXdYtYS_*>0Mxr;jH6n)XmJ{=@kHSqc!>c{a z#KYj>0V*e{G%Yfngmup6>!Dmb)}p{^n(W6rV^nEUu9!l9!(Lw#=0c{usc#-MY`zh3 zqf!KFb2CyR)l!+eR%613y12DBY5zm6(a2e><)%F69v08s>8O6eiH z?OJzcf0>3$v(TBVwV$IdvzZ4@Sx45Ns%&fN%Eu3iX69mY5;75t$ft5iJx`+6m;d5d zFlEp@C?}5ya&~q0_D~${#=+BLlme$et$OKAL;xr8Z&y!F;7rRZTcSvA_PI;()5It3 zP_;v1)ucC%fL^GspC_#8)-N!%2nq+rvCEnFQPkjgR*pbL2J|Un?p?wuo|S<-KiF3N z)A3+Ng8y3wM-rrZhxBo-)RenApw_<>*(VjsVt6<2KEQA^S#c;pKUQFoejMlRx5yy- zd$f&WQ~{qnn#Q%}uU2H_rxe@^rg|e`1t`$ZVY9H!FgFkrd7dIipNjhf z@XX2xM-%@n{YU0(peTS|9-`3iUDVF$8mLo3&63`HV|D%1V=Idfaa{2m&l|u%R_8#h zH3K}-6!8DtJ;lBeNMPijui1f4L0_K-N5ORH8FLVfaaXk%^J_0s6f=$?PkFgu&m%3m z8){RnRzV{J5V)#hgJ@D(WKkxAlYrUYd2s$_)IQJrF7e*L!Hkk}^8DwlZb5;gJrNxV zWUDy+z9#Tw24_%t5>`y(fH6(#!yCi6HQ?Y`=mAP!X9!>l{96&MNR%XqrFiqw(%N2{ zrGyTl&rd?+`KgAj2s5wJG+SwWL!2xH64Wb)Cm=`a^4!k$u z^_=w^gVY!Qm@#^)Ey0}u;4sjU@Tc(CxkYqh6fTBFh{V!SlNg?s>D2Sx*Y%g z@%z|is1l;9s+w7j1m9v(Z7WO@2ZHbG^; zQ$z`X$5Kykh7f|A4Gej0SX^(kKXzCwPd{E78;4{U z)m_t4$9{4wU84ss*(@Ot{%7LV$< zNuqwts+F&bArN*2r0zv4IRVm%WN4xU0~ausCqsd*0E4+U7#qF`7qVq4rUDei66H-G z2v!w;F^30$*n$3_Xk;Y0-jCaPpHE2c(PGPgMw&5G@5y7%Z7@A|Kr`OOTn0yiiT8VlnB_n{sqfs9?WEe9cM;8#p^EV*R;=kGI z+i;RNi7YK)-jFu`hUrGR>qFVFR#3{rQhdpNWRnGRH9NzhMi} z@9>WS?n6I=+Jn+#a)6YiDXrWytho9oS_dlxWz^y#1`%p`3cKR;4M2yjpmc^PAqbS5 zzJNc#-(3v>JJ$LYxB8dGiuChfW%0!Ut?H{8nT2>>{0PWfegOKpAD;|nqOZlN90mN% z1oS8u2nJ9dV9_=*Z7ut=(rzj94OQjEu;HsJ`xqDaB*9ZBBvWG;Y5hx}#ms694zuU7 z5ow&J9jsvrF-H<}yySIBcm@&-z+2Lb;%kb53*c>{4T@mk0GeKWz^eboOTN!+(cw%t zxL$~Xmm*TjHUz1acABKgq}*>=P(ey*n0_h>blYzh0iUwgun(n}Ne9U1Z;zbEGL%1< zdMssCP1ecpw5(D3QtO<=TTJyDUhlzESY3}4?rIElZU9xN75#Dns5uRws0Uup;EymX z6zQQa^}{uj+?95z1bAD`Xm!di zXt$iAg}b4yxj849cd}?A-b$R_hbNLYI8CmoAx=(mQQ}6ySuU^_zyQz4bi_ySiMX;A z6G|rvUI5)NHp@qA?V8?M+CYPBA6&=R{-~Y7Wr}L368tw*lOo~@}Ja5wYzJn6RjGWC;@2ceY4QH=m?qi2X>OSD{(LjvxC1v#&n{A*9+vP$Uy*PF8y)@nm`W)ZMyGj4IcM3mb8*>Ye@RqF@ot21^u?>-WgfR%;(5kY8{Zp>x=i3C+;9O&R0W4*rx<+b zjxB?ao(L(1E;2UIh4m2)+epql{{f}s`F{8Y6K4Tcx&UF;El`e@8@N&{02n2I^wScM zo()0K>Jxzwfch~-8}JI7S1TV||KrayVz3D`js1`TxFBNV#=4yaPL1ah`4g&MN`5ks z%$W>fqI44PY8OXZAN!{uF-mS5RE~^H7?gk|K{G-fB^gtW%@G;~vLA4W*=v&J%;j7} zCLrB|n80V>iH1e`|CRu`Q=8W7wp5C4q`FBZ10)dY4)`?wxsX6ZHl;&CPZT|9GMaA; z7IzwrLe* zd`w2!|24D_b|4=Ld`E{{f;)2t(fSrI|5I~Pspm0CVBKUW_2HVr49q3x)GP!=Vtd`b zmCwD+X7dh$`nFw`&T60BJ6Wz4F`?}el!YY^$nmobdqEpih42KKMw912jDQ5d)o0i? zta1(AEe*Cx4^6*!I;;k|Fv3o5Q22}!Vh^39`hbzg?`YBI6R-tX<#xNPr{+G5bmJq6 zYvH)g$z4f;4dJ@>=F_Cl65B$f#|=Gg9VmXnd)gL+s75fpAR?PBefryWRQ@ouY^g#T z-axx}-IA>73S8IE0+R!E^~&wX7-8D6n~X^Xch0?vJsj{{?coYRMTRD`?NGQZ2`-#d z;v+97FD{R45THub)#6?E4}^6@H*w!^d#P&=!Y94izA zM>_Hx(!C@*j?0Id|4XdkE$Yvm1%i7G)Y)uDs`O*fMapd2Mbr7<0m^Kz<3#}SqtQ`^ z{#b=i6(@dh4i9b^3%p7vdYXoO9DKyBoL5w=~ybm2&yb*=hmD%5)N%g;ms^s57 zZrJs}RD%JEPX0{)rz1E(4PF$b64Fx^8r-33&5dF^T@Qu9Sl|mxW_u?lm zMayPB?&y*iA-5`MxGt)F=vm6tY28Zb8J~SLwO= z-!})MRK9)4_=f}N`T-n(o*3~NUP)^yvt)LOoS|*SZF#kh4&|)j0#1F%(}{gPB_rMo z@~(4!4V+&52PXP~pdL+d#$V7975iz=2qLHl@8sGKIQhtNGt%NgrhshgOVz?^jb4fd zTQ8^|=oX@2@g(q$QK{oCvMD_7clu1LhblqKX2VZa&PjZ8XilAwO`4p+*5UQ>`FLCH zq$+)v5(^b$0Gzj|xFxqc!%G31C1_nB6(H>=P)_eE^>Sf)_L<;bJf-;OqeGhf3x95E zSKwcVoH0o65SleJE7T%ezMf0x3ot7nqfXo9El;8GfZ_346!qQ!_>uqeTFe)e%=+_p z;5beT%y9+E$98u0TNIt7bME5~^DRA3p6v$$1-L? z339KGd$jU;PV+F9__QNN=i)!{D-bRxhw0=sdGzSr)#snSbO&?lbyuXz{&OV32RdYm z`Qxwf!pJc*`NuTgIIdL6|KlrUN0|KF7AXlQU6ZOTJpTIy!jpy^y&gB$A^by4z_-}+ zw5n;!3d4rn{-3V@r?a%oQ+zB5eyjJs2zm!s{x>C-8qfPX+p7@c%p==-?j`0*)xpbywJM{^utEVp)LsN0)v+K_+v?{kpsD zH-C*sR}X!8R?L-VO$knV?Ml{qAY;piv8(8LbC9Epa2ua{CZ!T&qWlkI_b0nUeW*P& z$9*H$o8~LNq;oO6vs!pG+K2BQ>I?aYqIwiW?dwBRC7XZ+m^bI2s>zQ9Pd7B~MlluK zjn=Ma99FtMnvBx@ldk(JbXHRs7MpyY5Lj;#@ z?{h&Xw;P^W-{jlg_IG;7;Mta``=)ElD`|z}N__8JqErpaK^|ZC;(Kp;RV#+m1sSGR zKfS^nita8)6>EG*+g5kAfQeb->$zXmL8s21PU z^3;#j73)8vXEXwr8moht-wu%pR*AXx-AT*qr(wfy4UBJwjGXL=Jm(}R z)>U;*p$VPK7Rh3F?V-`gQ~b0>b@Pd%?8V@Z)sHB;a@4P3C8`UW?pPEDzn395Lw`amHa?g78vdV-q3hKtnoG@?M#W(tMOR(h zmp^1$H4Y0K4Gk;1`EYyZ%Ki|O&iYHVbV}@_g<**-Gv-mBbMJIb|WvO zrJ>7&gP;_vkpVG=J$pf+Rz3XJT(0+YD7V0*xZG1325Q=q>JHXT>5Rn2YcWunsI{Hr zxx~=Wl9mH2;*+!2VRieBxieJ{frir8K(+cuADf;3iz)g4^c{;QSx{Ykw$rF-e2u2- z!+(7gG*z9z-5hOSP{5w)rHz*d*(+oVZZY{Gv_u{5!*qs{6VAs2(jCda_H;32!Fl__ z9iy&i{nz}WkEvm?9$q+HO@k3F7O}5R2#r4Wy%5NIb5|eRjus}hD}c-Px*)rs-^Y^S zzjj5#(os<(o8(AET|;uB>2eaP41R_8!oP8>-&WEhE9<8>C9c*Dj$Diq^io%+RJ%p` zH>OnrlQ81@Z(s{JPH)@oiI*1-`X4c)2TuHRiV8=3;Af~OD8x5H7I`Kny}t<_NBAI~ zJQ2`Sq4c6%Y3^T)jl|vQb>s*LColl$@!!C+*LjI_`Ew%{1YI3~+WZ^u|M$NyeLtd3 YTx>m^`Tkc11pJ58Gtn*8cDeVz0MKFtJpcdz literal 0 HcmV?d00001 From 9bc4f7ad8d6c1b21c8bcf66c6a9264481f902595 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:00:14 +0100 Subject: [PATCH 08/14] Fix image centering --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fb45f342b..d88b906e9 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,10 @@ performance or outcome of some model trained on it. Some concepts of value depend on a specific model of interest, while others are model-agnostic. pyDVL focuses on model-dependent methods. -
+
+
best sample removal @@ -49,15 +50,17 @@ pyDVL focuses on model-dependent methods.
on best sample removal.
+
The **Influence Function** is an infinitesimal measure of the effect that single training points have over the parameters of a model, or any function thereof. In particular, in machine learning they are also used to compute the effect of training samples over individual test points. -
+
+
best sample removal @@ -66,6 +69,7 @@ of training samples over individual test points.
Highlighted points have flipped labels.
+
# Installation From 7295af2a6c93232c74f13307fafa54fe60c66a53 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:02:11 +0100 Subject: [PATCH 09/14] Prefer p over figcaption --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d88b906e9..1d04f3b76 100644 --- a/README.md +++ b/README.md @@ -39,17 +39,15 @@ value depend on a specific model of interest, while others are model-agnostic. pyDVL focuses on model-dependent methods.
-
best sample removal -
+

Comparison of different data valuation methods
on best sample removal. -

-
+

The **Influence Function** is an infinitesimal measure of the effect that single @@ -58,17 +56,15 @@ In particular, in machine learning they are also used to compute the effect of training samples over individual test points.
-
best sample removal -
+

Influences of input points with corrupted data.
Highlighted points have flipped labels. -

-
+

# Installation From 91870fa63cef44ab535505dfb7b0bf474880b52e Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:03:13 +0100 Subject: [PATCH 10/14] Cosmetic changes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1d04f3b76..540e5f9f6 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ pyDVL focuses on model-dependent methods. />

Comparison of different data valuation methods -
on best sample removal. + on best sample removal.

@@ -57,13 +57,13 @@ of training samples over individual test points.
best sample removal

Influences of input points with corrupted data. -
Highlighted points have flipped labels. + Highlighted points have flipped labels.

From c08299591418c69800c0fef1332802e0ad103e8d Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:04:20 +0100 Subject: [PATCH 11/14] Decrease image size --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 540e5f9f6..1c5de92e5 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ pyDVL focuses on model-dependent methods.
best sample removal @@ -57,7 +57,7 @@ of training samples over individual test points.
best sample removal @@ -278,10 +278,10 @@ contributions](CONTRIBUTING.md). # Papers -## Data Valuation - We currently implement the following papers: +## Data Valuation + - Castro, Javier, Daniel Gómez, and Juan Tejada. [Polynomial Calculation of the Shapley Value Based on Sampling](https://doi.org/10.1016/j.cor.2008.04.004). Computers & Operations Research, Selected papers presented at the Tenth @@ -328,8 +328,6 @@ We currently implement the following papers: ## Influence Functions -We currently implement the following papers: - - Koh, Pang Wei, and Percy Liang. [Understanding Black-Box Predictions via Influence Functions](http://proceedings.mlr.press/v70/koh17a.html). In Proceedings of the 34th International Conference on Machine Learning, From 860527200b20b1e01ee5918d9d35683fe30378ef Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:10:38 +0100 Subject: [PATCH 12/14] More centering --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c5de92e5..53e2be1d1 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,15 @@ performance or outcome of some model trained on it. Some concepts of value depend on a specific model of interest, while others are model-agnostic. pyDVL focuses on model-dependent methods. -
+
best sample removal -

+

Comparison of different data valuation methods on best sample removal.

@@ -55,13 +57,15 @@ training points have over the parameters of a model, or any function thereof. In particular, in machine learning they are also used to compute the effect of training samples over individual test points. -
+
best sample removal -

+

Influences of input points with corrupted data. Highlighted points have flipped labels.

From 2d23c1683329ae62da0bf1260e601d9393770584 Mon Sep 17 00:00:00 2001 From: Anes Benmerzoug Date: Mon, 11 Dec 2023 14:11:11 +0100 Subject: [PATCH 13/14] Even more centering --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 53e2be1d1..ecdd3ef91 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ performance or outcome of some model trained on it. Some concepts of value depend on a specific model of interest, while others are model-agnostic. pyDVL focuses on model-dependent methods. -
+
+
Date: Mon, 11 Dec 2023 14:20:56 +0100 Subject: [PATCH 14/14] Remove caching section from readme --- README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/README.md b/README.md index ecdd3ef91..90032ced3 100644 --- a/README.md +++ b/README.md @@ -259,21 +259,6 @@ The steps required to compute data values for your samples are: df = values.to_dataframe(column="data_value") ``` -## Caching - -pyDVL offers the possibility to cache certain results and -speed up computation. It uses [Memcached](https://memcached.org/) For that. - -You can run it either locally or, using -[Docker](https://www.docker.com/): - -```shell -docker container run --rm -p 11211:11211 --name pydvl-cache -d memcached:latest -``` - -You can read more in the -[documentation](https://pydvl.org/stable/getting-started/first-steps/#caching). - # Contributing Please open new issues for bugs, feature requests and extensions. You can read