From 4fea7aa2d78488003a4300d777b0e325585593b3 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:48:17 -0700 Subject: [PATCH 01/97] Update dataset.h --- include/LightGBM/dataset.h | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/include/LightGBM/dataset.h b/include/LightGBM/dataset.h index 825c5c6ebcf8..cc642973f769 100644 --- a/include/LightGBM/dataset.h +++ b/include/LightGBM/dataset.h @@ -213,6 +213,40 @@ class Metadata { } } + /*! + * \brief Get positions, if does not exist then return nullptr + * \return Pointer of positions + */ + inline const size_t* positions() const { + if (!positions_.empty()) { + return positions_.data(); + } + else { + return nullptr; + } + } + + /*! + * \brief Get position IDs, if does not exist then return nullptr + * \return Pointer of positions + */ + inline const std::string* position_ids() const { + if (!position_ids_.empty()) { + return position_ids_.data(); + } + else { + return nullptr; + } + } + + /*! + * \brief Get Number of different position IDs + * \return number of different position IDs + */ + inline const size_t num_position_ids() const { + return position_ids_.size(); + } + /*! * \brief Get data boundaries on queries, if not exists, will return nullptr * we assume data will order by query, @@ -289,6 +323,8 @@ class Metadata { private: /*! \brief Load wights from file */ void LoadWeights(); + /*! \brief Load positions from file */ + void LoadPositions(); /*! \brief Load query boundaries from file */ void LoadQueryBoundaries(); /*! \brief Calculate query weights from queries */ @@ -309,10 +345,16 @@ class Metadata { data_size_t num_data_; /*! \brief Number of weights, used to check correct weight file */ data_size_t num_weights_; + /*! \brief Number of positions, used to check correct position file */ + data_size_t num_positions_; /*! \brief Label data */ std::vector label_; /*! \brief Weights data */ std::vector weights_; + /*! \brief Positions data */ + std::vector positions_; + /*! \brief Position identifiers */ + std::vector position_ids_; /*! \brief Query boundaries */ std::vector query_boundaries_; /*! \brief Query weights */ @@ -328,6 +370,7 @@ class Metadata { /*! \brief mutex for threading safe call */ std::mutex mutex_; bool weight_load_from_file_; + bool position_load_from_file_; bool query_load_from_file_; bool init_score_load_from_file_; #ifdef USE_CUDA From e1bbbba5df0c871ba4d38233ac8f572fc673a640 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:52:11 -0700 Subject: [PATCH 02/97] Update metadata.cpp --- src/io/metadata.cpp | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 2a589fa24ef8..97aa0663b38e 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -28,6 +28,7 @@ void Metadata::Init(const char* data_filename) { // for lambdarank, it needs query data for partition data in distributed learning LoadQueryBoundaries(); LoadWeights(); + LoadPositions(); CalculateQueryWeights(); LoadInitialScore(data_filename_); } @@ -214,6 +215,13 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector 0 && num_positions_ != num_all_data) { + positions_.clear(); + num_positions_ = 0; + Log::Fatal("Positions size doesn't match data size"); + } + // get local weights + if (!positions_.empty()) { + auto old_positions = positions_; + num_positions_ = num_data_; + positions_ = std::vector(num_data_); + #pragma omp parallel for schedule(static, 512) + for (int i = 0; i < static_cast(used_data_indices.size()); ++i) { + positions_[i] = old_positions[used_data_indices[i]]; + } + old_positions.clear(); + } + } if (query_load_from_file_) { // check query boundries if (!query_boundaries_.empty() && query_boundaries_[num_queries_] != num_all_data) { @@ -528,6 +555,32 @@ void Metadata::LoadWeights() { weight_load_from_file_ = true; } +void Metadata::LoadPositions() { + num_positions_ = 0; + std::string position_filename(data_filename_); + // default position file name + position_filename.append(".position"); + TextReader reader(position_filename.c_str(), false); + reader.ReadAllLines(); + if (reader.Lines().empty()) { + return; + } + Log::Info("Loading positions..."); + num_positions_ = static_cast(reader.Lines().size()); + positions_ = std::vector(num_positions_); + position_ids_ = std::vector(); + std::unordered_map map_id2pos; + for (data_size_t i = 0; i < num_positions_; ++i) { + if (map_id2pos.count(reader.Lines()[i]) == 0) { + map_id2pos[reader.Lines()[i]] = position_ids_.size(); + position_ids_.push_back(reader.Lines()[i]); + } + positions_[i] = map_id2pos.at(reader.Lines()[i]); + } + position_load_from_file_ = true; +} + + void Metadata::LoadInitialScore(const std::string& data_filename) { num_init_score_ = 0; std::string init_score_filename(data_filename); From 6027716c7e3d359d80567ddc3232961499274745 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:53:41 -0700 Subject: [PATCH 03/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 63 +++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 653fc6e8609a..b10928171c66 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -37,6 +37,12 @@ class RankingObjective : public ObjectiveFunction { label_ = metadata.label(); // get weights weights_ = metadata.weights(); + // get positions + positions_ = metadata.positions(); + // get position ids + position_ids_ = metadata.position_ids(); + // get number of different position ids + num_position_ids_ = metadata.num_position_ids(); // get boundries query_boundaries_ = metadata.query_boundaries(); if (query_boundaries_ == nullptr) { @@ -62,6 +68,9 @@ class RankingObjective : public ObjectiveFunction { } } } + if (num_position_ids_ > 0) { + UpdatePositionBiasFactors(gradients, hessians); + } } virtual void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt, @@ -69,6 +78,8 @@ class RankingObjective : public ObjectiveFunction { const double* score, score_t* lambdas, score_t* hessians) const = 0; + virtual void UpdatePositionBiasFactors(const score_t* lambdas, const score_t* hessians) const {} + const char* GetName() const override = 0; std::string ToString() const override { @@ -88,6 +99,12 @@ class RankingObjective : public ObjectiveFunction { const label_t* label_; /*! \brief Pointer of weights */ const label_t* weights_; + /*! \brief Pointer of positions */ + const size_t* positions_; + /*! \brief Pointer of position IDs */ + const std::string* position_ids_; + /*! \brief Pointer of label */ + data_size_t num_position_ids_; /*! \brief Query boundaries */ const data_size_t* query_boundaries_; }; @@ -111,6 +128,7 @@ class LambdarankNDCG : public RankingObjective { if (sigmoid_ <= 0.0) { Log::Fatal("Sigmoid param %f should be greater than zero", sigmoid_); } + learning_rate_ = config.learning_rate; } explicit LambdarankNDCG(const std::vector& strs) @@ -135,6 +153,8 @@ class LambdarankNDCG : public RankingObjective { } // construct Sigmoid table to speed up Sigmoid transform ConstructSigmoidTable(); + // initialize position bias vectors + pos_biases_.resize(num_position_ids_, 0.0); } inline void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt, @@ -181,14 +201,18 @@ class LambdarankNDCG : public RankingObjective { } const data_size_t high = sorted_idx[high_rank]; const int high_label = static_cast(label[high]); - const double high_score = score[high]; + double high_score = score[high]; const double high_label_gain = label_gain_[high_label]; const double high_discount = DCGCalculator::GetDiscount(high_rank); const data_size_t low = sorted_idx[low_rank]; const int low_label = static_cast(label[low]); - const double low_score = score[low]; + double low_score = score[low]; const double low_label_gain = label_gain_[low_label]; const double low_discount = DCGCalculator::GetDiscount(low_rank); + if (num_position_ids_ > 0) { + high_score += pos_biases_[positions_[query_boundaries_[query_id] + high]]; + low_score += pos_biases_[positions_[query_boundaries_[query_id] + low]]; + } const double delta_score = high_score - low_score; @@ -253,9 +277,41 @@ class LambdarankNDCG : public RankingObjective { } } + void UpdatePositionBiasFactors(const score_t* lambdas, const score_t* hessians) const override { + std::vector bias_first_derivatives(num_position_ids_, 0.0); + std::vector bias_second_derivatives(num_position_ids_, 0.0); + for (data_size_t i = 0; i < num_data_; i++) { + // accumilate first derivatives of utility w.r.t. position bias factors, for each position + bias_first_derivatives[positions_[i]] -= lambdas[i]; + // accumilate second derivatives of utility w.r.t. position bias factors, for each position + bias_second_derivatives[positions_[i]] -= hessians[i]; + } + #pragma omp parallel for schedule(guided) + for (data_size_t i = 0; i < num_position_ids_; i++) { + // do Newton-Rhapson step to update position bias factors + pos_biases_[i] += learning_rate_ * bias_first_derivatives[i] / (std::abs(bias_second_derivatives[i]) + 0.001); + } + LogDebugPositionBiasFactors(); + } + const char* GetName() const override { return "lambdarank"; } protected: + void LogDebugPositionBiasFactors() const { + std::stringstream message_stream; + message_stream << std::setw(15) << "position" + << std::setw(15) << "bias_factor" + << std::endl; + Log::Debug(message_stream.str().c_str()); + message_stream.str(""); + + for (int i = 0; i < num_position_ids_; ++i) { + message_stream << std::setw(15) << position_ids_[i] + << std::setw(15) << pos_biases_[i]; + Log::Debug(message_stream.str().c_str()); + message_stream.str(""); + } + } /*! \brief Sigmoid param */ double sigmoid_; /*! \brief Normalize the lambdas or not */ @@ -276,6 +332,9 @@ class LambdarankNDCG : public RankingObjective { double max_sigmoid_input_ = 50; /*! \brief Factor that covert score to bin in Sigmoid table */ double sigmoid_table_idx_factor_; + /*! \brief Position bias factors */ + mutable std::vector pos_biases_; + double learning_rate_; }; /*! From 199d41233ffd70331adee8075a84fad1b2a731a7 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:44:06 -0700 Subject: [PATCH 04/97] Update metadata.cpp --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 97aa0663b38e..492031eef6e1 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -266,7 +266,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector Date: Fri, 16 Jun 2023 21:22:12 -0700 Subject: [PATCH 05/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 38 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index b10928171c66..a3e6ae7f85c0 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -42,7 +42,7 @@ class RankingObjective : public ObjectiveFunction { // get position ids position_ids_ = metadata.position_ids(); // get number of different position ids - num_position_ids_ = metadata.num_position_ids(); + num_position_ids_ = static_cast(metadata.num_position_ids()); // get boundries query_boundaries_ = metadata.query_boundaries(); if (query_boundaries_ == nullptr) { @@ -278,18 +278,38 @@ class LambdarankNDCG : public RankingObjective { } void UpdatePositionBiasFactors(const score_t* lambdas, const score_t* hessians) const override { - std::vector bias_first_derivatives(num_position_ids_, 0.0); - std::vector bias_second_derivatives(num_position_ids_, 0.0); + /// get number of threads + int num_threads = 1; + #pragma omp parallel + #pragma omp master + { + num_threads = omp_get_num_threads(); + } + // create per-thread buffers for first and second derivatives of utility w.r.t. position bias factors + std::vector bias_first_derivatives(num_position_ids_ * num_threads, 0.0); + std::vector bias_second_derivatives(num_position_ids_ * num_threads, 0.0); + #pragma omp parallel for schedule(guided) for (data_size_t i = 0; i < num_data_; i++) { - // accumilate first derivatives of utility w.r.t. position bias factors, for each position - bias_first_derivatives[positions_[i]] -= lambdas[i]; - // accumilate second derivatives of utility w.r.t. position bias factors, for each position - bias_second_derivatives[positions_[i]] -= hessians[i]; + // get thread ID + const int tid = omp_get_thread_num(); + size_t offset = static_cast(positions_[i] + tid * num_threads); + // accumulate first derivatives of utility w.r.t. position bias factors, for each position + bias_first_derivatives[offset] -= lambdas[i]; + // accumulate second derivatives of utility w.r.t. position bias factors, for each position + bias_second_derivatives[offset] -= hessians[i]; } #pragma omp parallel for schedule(guided) for (data_size_t i = 0; i < num_position_ids_; i++) { + double bias_first_derivative = 0.0; + double bias_second_derivative = 0.0; + // aggregate derivatives from per-thread buffers + for (int tid = 0; tid < num_threads; tid++) { + size_t offset = static_cast(i + tid * num_threads); + bias_first_derivative += bias_first_derivatives[offset]; + bias_second_derivative += bias_second_derivatives[offset]; + } // do Newton-Rhapson step to update position bias factors - pos_biases_[i] += learning_rate_ * bias_first_derivatives[i] / (std::abs(bias_second_derivatives[i]) + 0.001); + pos_biases_[i] += learning_rate_ * bias_first_derivative / (std::abs(bias_second_derivative) + 0.001); } LogDebugPositionBiasFactors(); } @@ -304,7 +324,6 @@ class LambdarankNDCG : public RankingObjective { << std::endl; Log::Debug(message_stream.str().c_str()); message_stream.str(""); - for (int i = 0; i < num_position_ids_; ++i) { message_stream << std::setw(15) << position_ids_[i] << std::setw(15) << pos_biases_[i]; @@ -334,6 +353,7 @@ class LambdarankNDCG : public RankingObjective { double sigmoid_table_idx_factor_; /*! \brief Position bias factors */ mutable std::vector pos_biases_; + /*! \brief Learning rate to update position bias factors */ double learning_rate_; }; From 93f67d16d03faeacc4057435859eec2ce8a2d54c Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:23:00 -0700 Subject: [PATCH 06/97] Update metadata.cpp --- src/io/metadata.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 492031eef6e1..df7c56c0f69b 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -15,7 +15,9 @@ Metadata::Metadata() { num_init_score_ = 0; num_data_ = 0; num_queries_ = 0; + num_positions_ = 0; weight_load_from_file_ = false; + position_load_from_file_ = false; query_load_from_file_ = false; init_score_load_from_file_ = false; #ifdef USE_CUDA @@ -266,7 +268,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector(); std::unordered_map map_id2pos; for (data_size_t i = 0; i < num_positions_; ++i) { - if (map_id2pos.count(reader.Lines()[i]) == 0) { - map_id2pos[reader.Lines()[i]] = position_ids_.size(); - position_ids_.push_back(reader.Lines()[i]); + std::string& line = reader.Lines()[i]; + if (map_id2pos.count(line) == 0) { + map_id2pos[line] = position_ids_.size(); + position_ids_.push_back(line); } - positions_[i] = map_id2pos.at(reader.Lines()[i]); + positions_[i] = map_id2pos.at(line); } position_load_from_file_ = true; } From c615b7df2264a6608e0bcd8e178e0d1a48e67990 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:23:57 -0700 Subject: [PATCH 07/97] Update dataset.h --- include/LightGBM/dataset.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/LightGBM/dataset.h b/include/LightGBM/dataset.h index cc642973f769..6492a75fa4f5 100644 --- a/include/LightGBM/dataset.h +++ b/include/LightGBM/dataset.h @@ -220,21 +220,19 @@ class Metadata { inline const size_t* positions() const { if (!positions_.empty()) { return positions_.data(); - } - else { + } else { return nullptr; } } /*! * \brief Get position IDs, if does not exist then return nullptr - * \return Pointer of positions + * \return Pointer of position IDs */ inline const std::string* position_ids() const { if (!position_ids_.empty()) { return position_ids_.data(); - } - else { + } else { return nullptr; } } From e374e9ae91ab2c38fcda5e0f8ab9a8a9bacdaab6 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:10:39 -0700 Subject: [PATCH 08/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index a3e6ae7f85c0..3f76042b04c8 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -292,7 +292,7 @@ class LambdarankNDCG : public RankingObjective { for (data_size_t i = 0; i < num_data_; i++) { // get thread ID const int tid = omp_get_thread_num(); - size_t offset = static_cast(positions_[i] + tid * num_threads); + size_t offset = static_cast(positions_[i] + tid * num_position_ids_); // accumulate first derivatives of utility w.r.t. position bias factors, for each position bias_first_derivatives[offset] -= lambdas[i]; // accumulate second derivatives of utility w.r.t. position bias factors, for each position @@ -304,7 +304,7 @@ class LambdarankNDCG : public RankingObjective { double bias_second_derivative = 0.0; // aggregate derivatives from per-thread buffers for (int tid = 0; tid < num_threads; tid++) { - size_t offset = static_cast(i + tid * num_threads); + size_t offset = static_cast(i + tid * num_position_ids_); bias_first_derivative += bias_first_derivatives[offset]; bias_second_derivative += bias_second_derivatives[offset]; } From 6c7b86ffc4b0100df58ca30f8fb8a880e71eadf0 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:11:56 -0700 Subject: [PATCH 09/97] Update metadata.cpp --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index df7c56c0f69b..c29299c3553a 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -268,7 +268,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector Date: Mon, 19 Jun 2023 20:12:59 -0700 Subject: [PATCH 10/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index e87cea3bfcbb..8d62f8bef5d4 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -746,6 +746,14 @@ def test_ranking_prediction_early_stopping(): with pytest.raises(AssertionError): np.testing.assert_allclose(ret_early, ret_early_more_strict) +def test_ranking_with_position_information(): + rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' + params = { + 'objective': 'lambdarank', + 'verbose': -1 + } + lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) + gbm = lgb.train(params, lgb_train, num_boost_round=50) def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) @@ -1265,7 +1273,7 @@ def test_feature_name_with_non_ascii(): X_train = np.random.normal(size=(100, 4)) y_train = np.random.random(100) # This has non-ascii strings. - feature_names = [u'F_零', u'F_一', u'F_二', u'F_三'] + feature_names = [u'F_零', u'F_一', u'F_二', u'F_三'] params = {'verbose': -1} lgb_train = lgb.Dataset(X_train, y_train) From 9f033ed693d9a6e1b260e5a2fe6d5ac2dff6513a Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:19:21 -0700 Subject: [PATCH 11/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 8d62f8bef5d4..c45a0ebc285d 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -1273,7 +1273,7 @@ def test_feature_name_with_non_ascii(): X_train = np.random.normal(size=(100, 4)) y_train = np.random.random(100) # This has non-ascii strings. - feature_names = [u'F_零', u'F_一', u'F_二', u'F_三'] + feature_names = [u'F_零', u'F_一', u'F_二', u'F_三'] params = {'verbose': -1} lgb_train = lgb.Dataset(X_train, y_train) From 50659d715a9e6073d7a946ec29cb2da32cfa47aa Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:43:15 -0700 Subject: [PATCH 12/97] Add files via upload --- examples/lambdarank/_rank.train.position | 3005 ++++++++++++++++++++++ 1 file changed, 3005 insertions(+) create mode 100644 examples/lambdarank/_rank.train.position diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position new file mode 100644 index 000000000000..34a73a651a30 --- /dev/null +++ b/examples/lambdarank/_rank.train.position @@ -0,0 +1,3005 @@ +pos_1 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_25 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_25 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_25 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_25 +pos_26 +pos_27 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_25 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_13 +pos_14 +pos_15 +pos_16 +pos_17 +pos_18 +pos_19 +pos_20 +pos_21 +pos_22 +pos_23 +pos_24 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 +pos_11 +pos_12 +pos_1 +pos_2 +pos_3 +pos_4 +pos_5 +pos_6 +pos_7 +pos_8 +pos_9 +pos_10 \ No newline at end of file From 45fbe8bd4f0da573df4551db684fa0f4ced0a6e0 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:44:51 -0700 Subject: [PATCH 13/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index c45a0ebc285d..9e2b7355c23f 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -25,6 +25,9 @@ make_synthetic_regression, mse_obj, pickle_and_unpickle_object, sklearn_multiclass_custom_objective, softmax) +from shutil import copyfile +from os import remove + decreasing_generator = itertools.count(0, -1) @@ -752,8 +755,10 @@ def test_ranking_with_position_information(): 'objective': 'lambdarank', 'verbose': -1 } + copyfile(str(rank_example_dir / '_rank.train.position', str(rank_example_dir / 'rank.train.position') lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) gbm = lgb.train(params, lgb_train, num_boost_round=50) + remove(str(rank_example_dir / 'rank.train.position') def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From 7579ce138275b705d4a8bd5e66ecab601377fca3 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Mon, 19 Jun 2023 22:06:32 -0700 Subject: [PATCH 14/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 9e2b7355c23f..49dd1ec110f4 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -755,10 +755,10 @@ def test_ranking_with_position_information(): 'objective': 'lambdarank', 'verbose': -1 } - copyfile(str(rank_example_dir / '_rank.train.position', str(rank_example_dir / 'rank.train.position') + copyfile(str(rank_example_dir / '_rank.train.position'), str(rank_example_dir / 'rank.train.position')) lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) gbm = lgb.train(params, lgb_train, num_boost_round=50) - remove(str(rank_example_dir / 'rank.train.position') + remove(str(rank_example_dir / 'rank.train.position')) def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From e74f75e43c82e4327a3df3c84911e14371e7c299 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:43:28 -0700 Subject: [PATCH 15/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 49dd1ec110f4..a2d71dd8415d 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -758,6 +758,11 @@ def test_ranking_with_position_information(): copyfile(str(rank_example_dir / '_rank.train.position'), str(rank_example_dir / 'rank.train.position')) lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) gbm = lgb.train(params, lgb_train, num_boost_round=50) + + open(str(rank_example_dir / 'rank.train.position'), 'w').close() + with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Positions size doesn't match data size"): + lgb.train(params, lgb_train, num_boost_round=50) + remove(str(rank_example_dir / 'rank.train.position')) def test_early_stopping(): From a1985e1af636659a4996599799def6682f698feb Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:22:14 -0700 Subject: [PATCH 16/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index a2d71dd8415d..ae7d93b287f7 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -759,7 +759,8 @@ def test_ranking_with_position_information(): lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) gbm = lgb.train(params, lgb_train, num_boost_round=50) - open(str(rank_example_dir / 'rank.train.position'), 'w').close() + with open(str(rank_example_dir / 'rank.train.position'), 'a') as file: + file.write('pos_1000') with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Positions size doesn't match data size"): lgb.train(params, lgb_train, num_boost_round=50) From 01189b0dbe5b7a607f3b8d069c5e7332ca7684c9 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:17:25 -0700 Subject: [PATCH 17/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index ae7d93b287f7..73edba722c2f 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -761,6 +761,7 @@ def test_ranking_with_position_information(): with open(str(rank_example_dir / 'rank.train.position'), 'a') as file: file.write('pos_1000') + file.close() with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Positions size doesn't match data size"): lgb.train(params, lgb_train, num_boost_round=50) From 065269b10f5e50018243d74e2793204c91af6294 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:04:42 -0700 Subject: [PATCH 18/97] Update _rank.train.position --- examples/lambdarank/_rank.train.position | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position index 34a73a651a30..ca448a98ad0e 100644 --- a/examples/lambdarank/_rank.train.position +++ b/examples/lambdarank/_rank.train.position @@ -3002,4 +3002,4 @@ pos_6 pos_7 pos_8 pos_9 -pos_10 \ No newline at end of file +pos_10 From b1d5529cee1f84aa10f0b17b4320480725783461 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:57:47 -0700 Subject: [PATCH 19/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 73edba722c2f..c2db561b0291 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -26,7 +26,6 @@ softmax) from shutil import copyfile -from os import remove decreasing_generator = itertools.count(0, -1) @@ -749,24 +748,24 @@ def test_ranking_prediction_early_stopping(): with pytest.raises(AssertionError): np.testing.assert_allclose(ret_early, ret_early_more_strict) -def test_ranking_with_position_information(): +def test_ranking_with_position_information(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' params = { 'objective': 'lambdarank', 'verbose': -1 } - copyfile(str(rank_example_dir / '_rank.train.position'), str(rank_example_dir / 'rank.train.position')) - lgb_train = lgb.Dataset(str(rank_example_dir / 'rank.train'), params=params) + copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) + copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) + copyfile(str(rank_example_dir / '_rank.train.position'), str(tmp_path / 'rank.train.position')) + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) gbm = lgb.train(params, lgb_train, num_boost_round=50) - with open(str(rank_example_dir / 'rank.train.position'), 'a') as file: + with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Positions size doesn't match data size"): lgb.train(params, lgb_train, num_boost_round=50) - remove(str(rank_example_dir / 'rank.train.position')) - def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) params = { From adcd240cba35ec56fb0e2e46a5964978b04e3bbf Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 19:44:39 -0700 Subject: [PATCH 20/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index c2db561b0291..4b05d50047ea 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -763,7 +763,7 @@ def test_ranking_with_position_information(tmp_path): with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() - with np.testing.assert_raises_regex(lgb.basic.LightGBMError, "Positions size doesn't match data size"): + with pytest.raises(lgb.basic.LightGBMError, "Positions size doesn't match data size"): lgb.train(params, lgb_train, num_boost_round=50) def test_early_stopping(): From 943e1a3c65fafb5cf4c2d0d5564c087a77a3efcd Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 20:52:42 -0700 Subject: [PATCH 21/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 4b05d50047ea..088e48f385d5 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -763,7 +763,7 @@ def test_ranking_with_position_information(tmp_path): with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() - with pytest.raises(lgb.basic.LightGBMError, "Positions size doesn't match data size"): + with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): lgb.train(params, lgb_train, num_boost_round=50) def test_early_stopping(): From 6333c77376a13f0ce7f7431c4849d52346ab6363 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 21:17:17 -0700 Subject: [PATCH 22/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 088e48f385d5..863b632610b7 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -763,8 +763,8 @@ def test_ranking_with_position_information(tmp_path): with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() - with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): - lgb.train(params, lgb_train, num_boost_round=50) + # with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): + lgb.train(params, lgb_train, num_boost_round=50) def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From 4fc927e18506e5b229d65b3690d770fe897a4d0f Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 22:00:06 -0700 Subject: [PATCH 23/97] Update _rank.train.position --- examples/lambdarank/_rank.train.position | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position index ca448a98ad0e..88aed8c3e148 100644 --- a/examples/lambdarank/_rank.train.position +++ b/examples/lambdarank/_rank.train.position @@ -3003,3 +3003,4 @@ pos_7 pos_8 pos_9 pos_10 +pos_1000 From f083d62628d337b12fa8ea40ada91297eafa639d Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 23:13:02 -0700 Subject: [PATCH 24/97] Update _rank.train.position --- examples/lambdarank/_rank.train.position | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position index 88aed8c3e148..ca448a98ad0e 100644 --- a/examples/lambdarank/_rank.train.position +++ b/examples/lambdarank/_rank.train.position @@ -3003,4 +3003,3 @@ pos_7 pos_8 pos_9 pos_10 -pos_1000 From 93238ac5ce2939e0b4dfc59efa30de27c633930c Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 20 Jun 2023 23:14:47 -0700 Subject: [PATCH 25/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 863b632610b7..abada79cb2c1 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -763,8 +763,9 @@ def test_ranking_with_position_information(tmp_path): with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() - # with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): - lgb.train(params, lgb_train, num_boost_round=50) + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): + lgb.train(params, lgb_train, num_boost_round=50) def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From ae647e45c002743eb789d327dbf681c884130d47 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:34:42 -0700 Subject: [PATCH 26/97] Update _rank.train.position --- examples/lambdarank/_rank.train.position | 3474 +++++++++++----------- 1 file changed, 1737 insertions(+), 1737 deletions(-) diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position index ca448a98ad0e..7aa6dcd9797e 100644 --- a/examples/lambdarank/_rank.train.position +++ b/examples/lambdarank/_rank.train.position @@ -1,3005 +1,3005 @@ pos_1 +pos_8 +pos_4 pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 +pos_12 pos_10 +pos_7 +pos_3 pos_11 -pos_12 pos_13 +pos_9 +pos_6 +pos_5 pos_1 -pos_2 +pos_5 pos_3 pos_4 -pos_5 +pos_2 pos_1 +pos_7 pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 pos_8 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 pos_6 +pos_3 +pos_6 +pos_16 +pos_1 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_17 -pos_18 pos_19 -pos_1 -pos_2 -pos_3 pos_4 pos_5 -pos_6 -pos_7 +pos_12 +pos_2 +pos_17 +pos_3 pos_8 +pos_15 +pos_18 +pos_11 pos_9 pos_10 +pos_13 +pos_7 +pos_6 +pos_8 +pos_2 +pos_9 pos_11 +pos_4 pos_12 pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 +pos_3 pos_10 -pos_11 +pos_3 pos_12 -pos_13 -pos_14 -pos_15 -pos_16 +pos_11 +pos_9 pos_17 -pos_18 pos_1 pos_2 -pos_3 +pos_8 pos_4 +pos_18 +pos_16 +pos_7 +pos_15 +pos_13 pos_5 -pos_1 -pos_2 +pos_6 +pos_14 +pos_10 pos_3 +pos_2 +pos_1 pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 +pos_13 +pos_5 pos_10 pos_11 +pos_7 +pos_4 pos_12 -pos_13 +pos_6 +pos_8 pos_14 pos_1 pos_2 pos_3 -pos_4 +pos_9 pos_5 -pos_6 -pos_7 +pos_13 pos_8 +pos_2 +pos_11 +pos_7 pos_9 pos_10 -pos_11 pos_12 -pos_13 -pos_1 -pos_2 +pos_6 pos_3 +pos_1 pos_4 pos_5 -pos_6 +pos_4 +pos_2 pos_7 -pos_8 pos_1 -pos_2 +pos_8 +pos_6 pos_3 -pos_4 pos_5 +pos_4 +pos_1 pos_6 -pos_7 +pos_3 +pos_2 pos_8 pos_9 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 -pos_6 pos_7 -pos_8 +pos_14 +pos_4 pos_9 -pos_10 -pos_11 pos_12 -pos_13 -pos_14 -pos_15 +pos_7 pos_16 pos_1 +pos_10 +pos_11 +pos_13 pos_2 +pos_15 pos_3 +pos_8 +pos_6 +pos_5 pos_4 +pos_1 +pos_10 pos_5 -pos_6 +pos_9 +pos_3 +pos_11 pos_7 pos_8 -pos_9 +pos_2 +pos_6 +pos_19 pos_10 -pos_11 +pos_3 pos_1 +pos_17 +pos_9 +pos_18 pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 +pos_15 pos_8 -pos_9 -pos_10 pos_11 +pos_21 pos_12 -pos_13 pos_14 -pos_15 pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 pos_13 -pos_14 -pos_1 +pos_20 pos_2 pos_3 -pos_4 -pos_5 -pos_6 pos_7 -pos_8 pos_9 -pos_10 pos_11 -pos_12 +pos_4 pos_13 +pos_10 +pos_8 +pos_12 +pos_5 +pos_1 pos_14 +pos_6 +pos_20 +pos_5 +pos_7 +pos_4 pos_15 +pos_9 +pos_6 +pos_11 pos_16 +pos_1 +pos_8 +pos_13 pos_17 -pos_18 -pos_19 -pos_20 pos_21 -pos_1 +pos_19 +pos_14 pos_2 +pos_18 pos_3 +pos_12 +pos_10 pos_4 -pos_5 +pos_1 +pos_9 pos_6 -pos_7 pos_8 -pos_9 -pos_1 +pos_7 +pos_5 pos_2 pos_3 -pos_4 +pos_12 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_1 -pos_2 pos_3 +pos_8 pos_4 -pos_5 pos_6 +pos_1 +pos_2 +pos_14 pos_7 -pos_8 pos_9 -pos_10 +pos_13 pos_11 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 pos_6 -pos_7 -pos_8 +pos_5 pos_9 +pos_4 pos_10 -pos_11 -pos_12 -pos_13 +pos_2 +pos_3 +pos_1 +pos_7 +pos_8 pos_14 -pos_15 -pos_16 +pos_7 pos_17 pos_18 -pos_19 -pos_20 pos_1 -pos_2 -pos_3 +pos_8 pos_4 +pos_10 +pos_20 +pos_3 +pos_12 +pos_2 +pos_15 +pos_16 +pos_19 +pos_11 +pos_13 +pos_9 pos_5 pos_6 pos_7 -pos_8 pos_9 +pos_4 +pos_13 pos_10 -pos_11 pos_12 -pos_13 -pos_14 -pos_15 -pos_16 +pos_6 pos_17 -pos_18 pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 +pos_18 +pos_16 +pos_11 +pos_14 +pos_2 pos_8 +pos_15 +pos_5 pos_9 -pos_10 -pos_11 pos_12 +pos_11 +pos_8 +pos_10 pos_13 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 +pos_1 pos_6 pos_7 +pos_5 +pos_4 +pos_3 +pos_19 +pos_13 +pos_12 +pos_17 +pos_14 +pos_6 pos_8 pos_9 +pos_4 pos_10 +pos_1 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 +pos_2 +pos_7 +pos_20 pos_16 -pos_17 pos_18 -pos_19 -pos_20 -pos_1 -pos_2 pos_3 -pos_4 pos_5 +pos_15 +pos_3 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 pos_20 -pos_21 -pos_22 pos_1 pos_2 -pos_3 +pos_8 +pos_15 pos_4 +pos_12 +pos_18 pos_5 -pos_6 +pos_10 +pos_17 +pos_16 +pos_13 +pos_22 pos_7 -pos_8 +pos_19 +pos_11 +pos_21 pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 pos_15 +pos_10 +pos_9 +pos_14 +pos_13 +pos_6 +pos_19 +pos_5 pos_16 +pos_3 +pos_21 +pos_12 +pos_4 +pos_2 pos_17 pos_18 -pos_19 -pos_20 -pos_21 pos_22 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_20 pos_7 pos_8 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 pos_6 pos_7 +pos_12 +pos_3 +pos_2 pos_8 -pos_9 +pos_13 +pos_5 pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_15 +pos_9 pos_7 pos_8 -pos_9 +pos_3 +pos_14 +pos_5 +pos_12 pos_10 pos_1 +pos_16 +pos_6 pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 -pos_10 +pos_17 pos_11 -pos_12 pos_13 -pos_1 -pos_2 pos_3 +pos_2 +pos_8 +pos_10 pos_4 -pos_5 +pos_7 pos_6 +pos_9 +pos_5 +pos_1 +pos_10 pos_7 +pos_4 +pos_6 +pos_2 +pos_13 +pos_3 pos_8 +pos_1 pos_9 -pos_10 pos_11 pos_12 -pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 +pos_4 pos_11 pos_12 -pos_13 +pos_10 +pos_6 +pos_5 +pos_9 +pos_3 pos_1 pos_2 +pos_8 +pos_1 +pos_8 pos_3 -pos_4 -pos_5 pos_6 -pos_7 -pos_8 pos_9 pos_10 +pos_7 +pos_4 +pos_5 pos_11 -pos_12 pos_13 -pos_1 pos_2 +pos_12 +pos_2 +pos_9 +pos_10 +pos_6 pos_3 +pos_11 pos_4 +pos_8 +pos_13 +pos_1 +pos_12 +pos_7 pos_5 pos_6 +pos_23 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 +pos_19 pos_14 -pos_15 +pos_2 +pos_4 pos_16 -pos_17 +pos_10 +pos_8 +pos_3 +pos_1 pos_18 -pos_19 +pos_15 +pos_12 +pos_17 +pos_22 pos_20 pos_21 -pos_22 -pos_23 -pos_1 -pos_2 -pos_3 -pos_4 +pos_13 pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 +pos_4 +pos_8 +pos_11 pos_18 +pos_6 pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 +pos_7 +pos_5 +pos_3 pos_12 +pos_16 +pos_15 +pos_17 +pos_9 pos_13 -pos_1 +pos_14 pos_2 pos_3 -pos_4 pos_5 +pos_11 +pos_10 pos_6 -pos_7 +pos_4 pos_8 +pos_13 +pos_1 +pos_12 pos_9 +pos_7 +pos_17 pos_10 -pos_11 +pos_14 +pos_4 pos_12 +pos_19 +pos_3 +pos_9 +pos_6 +pos_1 pos_13 -pos_14 +pos_7 +pos_11 +pos_8 pos_15 pos_16 -pos_17 pos_18 -pos_19 -pos_20 -pos_1 pos_2 +pos_20 +pos_5 pos_3 -pos_4 pos_5 -pos_6 pos_7 -pos_8 pos_9 -pos_10 -pos_11 pos_12 -pos_1 +pos_6 pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 +pos_11 +pos_10 pos_8 +pos_1 +pos_3 +pos_7 +pos_4 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 +pos_2 pos_15 +pos_5 +pos_13 pos_16 +pos_8 pos_17 +pos_14 +pos_10 +pos_21 +pos_6 pos_18 +pos_1 +pos_12 +pos_11 pos_19 pos_20 -pos_21 pos_22 -pos_1 -pos_2 pos_3 -pos_4 -pos_5 +pos_12 +pos_1 pos_6 pos_7 -pos_8 -pos_9 -pos_10 +pos_4 pos_11 -pos_12 -pos_13 pos_14 -pos_1 +pos_10 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 -pos_7 pos_8 +pos_5 pos_9 +pos_13 +pos_13 +pos_7 pos_10 +pos_3 pos_11 -pos_12 -pos_13 +pos_6 pos_1 -pos_2 -pos_3 pos_4 pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 +pos_8 +pos_2 pos_12 -pos_13 -pos_14 +pos_2 pos_15 -pos_16 -pos_17 pos_18 -pos_19 +pos_6 pos_20 -pos_21 -pos_22 +pos_12 pos_23 -pos_1 -pos_2 -pos_3 pos_4 +pos_9 +pos_11 +pos_10 pos_5 -pos_6 +pos_1 +pos_3 +pos_14 +pos_13 pos_7 +pos_17 +pos_19 +pos_22 +pos_21 +pos_16 pos_8 pos_9 +pos_4 +pos_1 +pos_8 +pos_13 +pos_3 pos_10 -pos_11 pos_12 -pos_13 -pos_1 +pos_7 +pos_6 pos_2 -pos_3 -pos_4 +pos_5 +pos_11 pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 +pos_3 +pos_4 pos_11 -pos_12 +pos_2 +pos_8 pos_13 +pos_7 +pos_10 pos_14 +pos_12 +pos_9 pos_1 +pos_8 +pos_12 pos_2 pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_12 pos_13 +pos_11 pos_14 pos_1 +pos_5 +pos_9 +pos_6 +pos_7 pos_2 -pos_3 +pos_1 pos_4 pos_5 -pos_1 -pos_2 +pos_3 pos_3 pos_4 +pos_11 +pos_8 +pos_1 +pos_13 pos_5 +pos_10 +pos_12 pos_6 pos_7 -pos_8 +pos_2 pos_9 -pos_10 +pos_6 +pos_2 +pos_7 pos_11 -pos_12 pos_13 +pos_5 +pos_9 +pos_8 +pos_15 +pos_12 pos_1 -pos_2 -pos_3 +pos_10 +pos_14 pos_4 +pos_3 +pos_2 pos_5 +pos_11 +pos_4 +pos_14 +pos_3 pos_6 -pos_7 +pos_13 +pos_10 pos_8 +pos_7 +pos_12 +pos_1 pos_9 -pos_10 -pos_11 pos_12 -pos_13 pos_14 -pos_15 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 pos_6 -pos_7 pos_8 pos_9 +pos_7 +pos_3 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 pos_1 +pos_5 pos_2 -pos_3 +pos_13 +pos_11 pos_4 pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 +pos_6 pos_12 -pos_13 -pos_14 pos_15 pos_16 -pos_1 pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 +pos_1 +pos_13 +pos_14 +pos_8 pos_11 +pos_4 pos_12 +pos_8 pos_13 +pos_2 +pos_9 pos_14 -pos_15 +pos_7 pos_16 +pos_11 +pos_10 +pos_15 pos_1 -pos_2 +pos_6 pos_3 -pos_4 pos_5 +pos_13 +pos_14 pos_6 +pos_4 pos_7 -pos_8 pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 +pos_10 pos_1 pos_2 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 +pos_15 pos_8 +pos_12 +pos_5 pos_9 -pos_10 pos_11 -pos_12 +pos_19 +pos_18 pos_13 pos_14 -pos_15 -pos_16 +pos_6 +pos_7 +pos_5 +pos_4 +pos_12 +pos_1 pos_17 -pos_18 -pos_19 pos_20 -pos_21 -pos_1 -pos_2 +pos_10 +pos_15 pos_3 +pos_8 +pos_2 +pos_21 +pos_16 pos_4 -pos_5 +pos_8 pos_6 +pos_12 pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 +pos_1 pos_17 -pos_18 -pos_19 pos_20 +pos_13 +pos_16 +pos_3 pos_21 +pos_19 +pos_5 +pos_18 pos_22 -pos_1 +pos_15 +pos_14 +pos_11 +pos_9 pos_2 pos_3 -pos_4 -pos_5 +pos_1 pos_6 -pos_7 pos_8 -pos_9 -pos_10 -pos_1 pos_2 -pos_3 -pos_4 pos_5 -pos_6 pos_7 -pos_8 pos_9 +pos_4 +pos_10 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 pos_15 -pos_16 pos_17 -pos_18 -pos_19 pos_20 -pos_21 -pos_22 -pos_1 -pos_2 -pos_3 pos_4 +pos_13 pos_5 +pos_2 +pos_18 +pos_9 pos_6 +pos_11 +pos_1 +pos_21 +pos_19 +pos_22 pos_7 pos_8 -pos_9 -pos_10 -pos_11 pos_12 -pos_13 pos_14 -pos_15 pos_16 -pos_17 -pos_18 -pos_1 +pos_3 pos_2 +pos_9 +pos_14 pos_3 +pos_1 +pos_16 +pos_8 +pos_5 pos_4 +pos_15 +pos_17 +pos_12 +pos_11 +pos_10 +pos_18 +pos_7 +pos_6 +pos_13 +pos_18 +pos_23 +pos_20 +pos_13 +pos_25 pos_5 pos_6 -pos_7 +pos_21 +pos_1 pos_8 +pos_7 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 pos_16 +pos_12 pos_17 -pos_18 +pos_2 pos_19 -pos_20 -pos_21 -pos_22 -pos_23 pos_24 -pos_25 -pos_1 -pos_2 +pos_15 +pos_10 pos_3 +pos_22 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 +pos_11 +pos_14 pos_9 +pos_6 +pos_1 pos_10 -pos_11 +pos_4 +pos_8 +pos_7 pos_12 -pos_13 -pos_14 -pos_15 pos_16 -pos_1 -pos_2 +pos_5 +pos_11 +pos_14 pos_3 +pos_2 +pos_15 +pos_13 +pos_11 pos_4 -pos_5 -pos_6 +pos_10 pos_7 -pos_8 pos_9 -pos_10 -pos_11 -pos_12 -pos_1 +pos_6 pos_2 +pos_8 +pos_12 pos_3 -pos_4 +pos_1 pos_5 -pos_6 -pos_7 pos_8 +pos_6 +pos_12 pos_9 +pos_3 pos_10 pos_11 -pos_12 -pos_1 -pos_2 -pos_3 pos_4 +pos_2 +pos_1 +pos_7 pos_5 pos_6 -pos_7 +pos_3 +pos_2 +pos_10 pos_8 +pos_1 +pos_15 pos_9 -pos_10 -pos_11 -pos_12 +pos_4 pos_13 pos_14 -pos_15 -pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 -pos_10 pos_11 pos_12 -pos_13 +pos_7 pos_14 -pos_15 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 -pos_6 -pos_7 +pos_13 +pos_11 pos_8 -pos_9 +pos_7 pos_10 -pos_11 +pos_9 +pos_5 +pos_3 +pos_15 +pos_2 +pos_6 pos_12 -pos_13 +pos_4 +pos_22 pos_14 +pos_12 +pos_11 +pos_2 +pos_24 +pos_7 pos_15 -pos_16 -pos_17 -pos_18 pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_24 -pos_25 -pos_1 -pos_2 -pos_3 -pos_4 pos_5 +pos_3 pos_6 -pos_7 -pos_8 +pos_25 pos_9 +pos_4 +pos_23 +pos_21 +pos_20 +pos_8 +pos_16 +pos_18 +pos_1 +pos_13 +pos_17 pos_10 -pos_11 -pos_12 pos_13 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 -pos_6 -pos_7 pos_8 -pos_9 +pos_10 pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 +pos_5 pos_11 +pos_7 pos_12 -pos_1 +pos_6 +pos_4 pos_2 -pos_3 pos_4 -pos_5 -pos_6 pos_7 -pos_8 pos_1 -pos_2 pos_3 -pos_4 -pos_5 +pos_9 pos_6 -pos_7 pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_1 +pos_5 pos_2 -pos_3 +pos_7 pos_4 -pos_5 +pos_3 pos_6 -pos_7 pos_8 +pos_12 pos_9 +pos_5 +pos_2 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_24 -pos_25 pos_1 -pos_2 -pos_3 -pos_4 +pos_11 +pos_8 pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_1 -pos_2 pos_3 -pos_4 -pos_5 pos_6 -pos_7 -pos_8 +pos_4 +pos_1 +pos_2 pos_9 +pos_16 pos_10 +pos_5 +pos_13 +pos_2 pos_11 +pos_3 +pos_1 pos_12 -pos_13 pos_14 +pos_4 +pos_8 +pos_7 +pos_6 pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_24 -pos_1 -pos_2 -pos_3 pos_4 pos_5 -pos_6 +pos_23 +pos_14 +pos_3 +pos_22 pos_7 +pos_20 pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_1 +pos_18 +pos_25 pos_2 -pos_3 -pos_4 -pos_5 +pos_17 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 +pos_15 pos_12 pos_13 -pos_14 -pos_15 -pos_16 +pos_10 pos_1 -pos_2 -pos_3 -pos_4 +pos_24 +pos_11 +pos_19 +pos_9 +pos_21 +pos_16 +pos_16 +pos_13 pos_5 -pos_6 -pos_7 -pos_8 pos_9 pos_10 -pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 +pos_17 +pos_12 +pos_19 pos_8 -pos_9 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 +pos_18 pos_11 -pos_12 -pos_13 -pos_14 +pos_2 +pos_1 pos_15 -pos_16 +pos_19 pos_17 -pos_1 -pos_2 -pos_3 -pos_4 -pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 pos_12 -pos_13 -pos_14 pos_15 -pos_1 +pos_14 pos_2 -pos_3 -pos_4 +pos_24 pos_5 -pos_6 -pos_7 +pos_13 +pos_21 +pos_10 +pos_9 +pos_8 pos_1 -pos_2 -pos_3 pos_4 -pos_5 -pos_6 +pos_11 +pos_22 pos_7 -pos_8 -pos_9 -pos_1 -pos_2 +pos_18 pos_3 -pos_4 -pos_5 +pos_20 +pos_23 +pos_16 pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 pos_12 -pos_13 -pos_14 -pos_15 -pos_1 +pos_7 pos_2 pos_3 -pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 +pos_4 pos_11 -pos_12 -pos_13 -pos_14 +pos_8 pos_1 -pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_12 -pos_13 pos_14 +pos_12 pos_15 +pos_13 +pos_2 +pos_5 pos_16 pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_1 +pos_9 +pos_7 +pos_6 pos_2 -pos_3 -pos_4 pos_5 -pos_6 pos_7 -pos_8 -pos_1 -pos_2 +pos_6 pos_3 +pos_9 pos_4 -pos_5 -pos_6 -pos_7 +pos_10 +pos_1 pos_8 pos_9 +pos_6 +pos_8 +pos_5 +pos_7 +pos_3 +pos_15 +pos_13 pos_10 +pos_2 pos_11 -pos_12 -pos_13 +pos_1 pos_14 -pos_15 +pos_4 pos_16 -pos_17 -pos_1 +pos_12 +pos_9 +pos_5 pos_2 pos_3 -pos_4 -pos_5 pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 pos_1 -pos_2 -pos_3 pos_4 +pos_8 +pos_9 pos_5 pos_6 +pos_3 +pos_17 +pos_12 pos_7 -pos_8 -pos_9 pos_10 +pos_4 +pos_1 +pos_2 pos_11 -pos_12 -pos_13 pos_14 +pos_8 pos_15 +pos_13 pos_16 -pos_17 -pos_18 -pos_1 -pos_2 -pos_3 -pos_4 +pos_12 +pos_8 pos_5 +pos_4 pos_6 +pos_1 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 pos_13 +pos_11 +pos_9 pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_1 pos_2 +pos_10 +pos_15 pos_3 -pos_4 +pos_7 pos_5 pos_6 +pos_4 +pos_2 +pos_3 +pos_1 +pos_3 +pos_9 +pos_2 pos_7 +pos_5 +pos_4 pos_8 +pos_6 +pos_1 pos_9 pos_10 -pos_1 -pos_2 -pos_3 pos_4 pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 pos_11 pos_12 -pos_1 pos_2 +pos_1 +pos_8 +pos_14 +pos_15 +pos_7 pos_3 -pos_4 -pos_5 -pos_6 +pos_13 pos_7 -pos_8 pos_9 -pos_10 +pos_6 pos_11 pos_12 pos_1 -pos_2 +pos_14 pos_3 +pos_13 +pos_5 +pos_2 +pos_10 +pos_8 pos_4 pos_1 -pos_2 pos_3 +pos_2 +pos_10 +pos_11 +pos_9 pos_4 +pos_8 +pos_12 +pos_14 +pos_16 +pos_6 +pos_15 pos_5 +pos_7 +pos_13 +pos_13 pos_6 +pos_2 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 +pos_5 +pos_3 pos_12 -pos_13 +pos_17 pos_14 +pos_10 +pos_9 pos_1 -pos_2 -pos_3 +pos_16 +pos_8 +pos_15 pos_4 -pos_5 +pos_11 pos_6 -pos_7 pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_1 -pos_2 pos_3 pos_4 +pos_2 +pos_7 +pos_1 pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 pos_1 +pos_15 pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 pos_8 +pos_6 pos_9 -pos_10 -pos_11 -pos_12 pos_13 pos_14 -pos_15 +pos_4 +pos_10 +pos_3 pos_16 pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_24 -pos_25 -pos_26 -pos_27 +pos_12 +pos_5 +pos_7 pos_1 -pos_2 +pos_10 pos_3 +pos_9 pos_4 -pos_5 -pos_6 -pos_7 pos_8 -pos_9 -pos_10 -pos_11 +pos_2 pos_12 +pos_11 +pos_6 +pos_17 pos_13 +pos_8 +pos_2 +pos_4 +pos_9 +pos_18 pos_14 +pos_12 pos_15 +pos_10 pos_16 +pos_5 pos_1 -pos_2 +pos_11 +pos_7 +pos_6 pos_3 -pos_4 -pos_5 +pos_11 +pos_23 +pos_15 +pos_9 +pos_8 pos_6 +pos_14 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 +pos_5 +pos_4 +pos_16 +pos_21 pos_12 +pos_10 pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_18 pos_19 -pos_20 -pos_1 pos_2 +pos_22 +pos_17 +pos_1 +pos_20 pos_3 -pos_4 +pos_18 +pos_10 pos_5 -pos_6 pos_7 +pos_3 +pos_6 pos_8 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 pos_1 pos_2 -pos_3 pos_4 -pos_5 +pos_4 +pos_1 +pos_3 pos_6 -pos_7 -pos_8 pos_9 +pos_5 +pos_8 pos_10 +pos_12 +pos_7 +pos_2 pos_11 +pos_8 +pos_11 +pos_4 +pos_10 pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 +pos_7 +pos_1 +pos_3 +pos_9 +pos_6 +pos_5 +pos_2 +pos_4 +pos_3 +pos_2 pos_1 +pos_6 +pos_11 +pos_12 +pos_4 +pos_14 pos_2 pos_3 -pos_4 -pos_5 -pos_6 +pos_1 pos_7 +pos_10 pos_8 +pos_5 pos_9 -pos_10 -pos_11 -pos_12 pos_13 -pos_1 -pos_2 +pos_7 pos_3 pos_4 -pos_5 pos_6 -pos_7 pos_8 -pos_9 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 +pos_9 pos_1 +pos_12 +pos_5 pos_2 -pos_3 +pos_11 pos_4 +pos_3 +pos_10 pos_5 -pos_6 +pos_1 +pos_2 +pos_15 pos_7 pos_8 -pos_9 -pos_10 +pos_14 +pos_6 pos_11 +pos_9 pos_12 pos_13 +pos_7 +pos_23 +pos_12 +pos_19 +pos_1 +pos_27 +pos_8 +pos_21 +pos_17 +pos_5 +pos_11 pos_14 -pos_15 +pos_4 pos_16 -pos_17 -pos_1 +pos_6 +pos_9 +pos_3 +pos_25 +pos_22 +pos_10 +pos_18 +pos_20 +pos_26 pos_2 +pos_15 +pos_24 +pos_13 pos_3 -pos_4 -pos_5 +pos_16 pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_1 pos_2 -pos_3 -pos_4 +pos_11 pos_5 -pos_6 +pos_15 +pos_13 pos_7 +pos_4 pos_8 pos_9 -pos_10 -pos_11 -pos_12 pos_1 -pos_2 +pos_15 pos_3 +pos_16 +pos_18 +pos_2 pos_4 +pos_7 pos_5 +pos_12 +pos_19 pos_6 -pos_7 -pos_8 +pos_20 pos_9 +pos_13 pos_10 pos_11 -pos_12 -pos_13 +pos_17 pos_14 -pos_15 pos_1 +pos_8 +pos_12 +pos_8 +pos_5 +pos_13 +pos_11 +pos_4 pos_2 +pos_1 pos_3 -pos_4 -pos_5 +pos_10 +pos_9 pos_6 pos_7 +pos_4 +pos_6 +pos_14 +pos_2 +pos_15 +pos_18 +pos_11 +pos_5 pos_8 -pos_9 pos_10 -pos_11 -pos_12 +pos_9 +pos_19 +pos_17 +pos_16 pos_13 -pos_14 +pos_12 pos_1 -pos_2 +pos_7 pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 +pos_11 pos_9 pos_10 -pos_11 -pos_12 +pos_6 pos_13 -pos_14 -pos_1 +pos_12 +pos_5 pos_2 +pos_7 +pos_1 pos_3 -pos_4 -pos_5 -pos_6 +pos_8 pos_7 +pos_11 +pos_5 +pos_3 pos_8 +pos_6 +pos_16 +pos_1 +pos_4 +pos_15 +pos_2 +pos_13 +pos_17 +pos_12 +pos_14 pos_9 pos_10 -pos_11 -pos_12 -pos_13 +pos_1 pos_14 +pos_13 pos_15 +pos_8 +pos_12 pos_16 +pos_6 pos_17 -pos_18 -pos_19 -pos_1 -pos_2 -pos_3 +pos_11 pos_4 pos_5 -pos_6 -pos_7 -pos_8 +pos_3 pos_9 pos_10 -pos_11 -pos_12 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 pos_7 +pos_6 +pos_5 +pos_16 pos_8 -pos_9 -pos_10 +pos_7 +pos_3 pos_11 -pos_12 -pos_13 +pos_10 +pos_4 pos_14 pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_23 -pos_1 +pos_12 +pos_13 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_1 +pos_9 pos_7 pos_8 -pos_9 +pos_6 +pos_1 +pos_5 +pos_4 pos_10 -pos_11 pos_12 +pos_3 +pos_9 +pos_11 +pos_2 pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_1 +pos_6 +pos_9 +pos_4 pos_2 pos_3 -pos_4 +pos_11 +pos_15 pos_5 -pos_6 pos_7 +pos_14 +pos_12 +pos_1 +pos_10 pos_8 +pos_13 +pos_2 pos_9 -pos_10 -pos_11 +pos_8 +pos_7 pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_1 -pos_2 -pos_3 +pos_11 pos_4 pos_5 pos_6 -pos_7 -pos_8 -pos_9 +pos_10 +pos_3 pos_1 -pos_2 pos_3 -pos_4 +pos_1 pos_5 -pos_6 +pos_10 +pos_12 pos_7 -pos_8 pos_9 -pos_10 +pos_6 +pos_14 +pos_4 pos_11 -pos_12 +pos_8 pos_13 +pos_2 +pos_11 +pos_2 +pos_4 +pos_6 +pos_12 pos_14 pos_15 pos_16 -pos_17 +pos_13 pos_18 +pos_17 +pos_8 +pos_9 +pos_7 pos_19 -pos_20 -pos_21 -pos_22 -pos_23 pos_1 -pos_2 pos_3 -pos_4 pos_5 +pos_10 pos_6 pos_7 +pos_11 pos_8 pos_9 +pos_2 pos_10 -pos_11 +pos_5 pos_1 -pos_2 pos_3 pos_4 +pos_12 pos_5 -pos_6 -pos_7 -pos_8 +pos_4 +pos_14 pos_9 pos_10 +pos_8 +pos_7 +pos_19 +pos_21 pos_11 -pos_12 -pos_13 -pos_14 +pos_18 +pos_16 +pos_20 pos_15 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 +pos_13 +pos_12 pos_6 -pos_7 -pos_8 -pos_1 +pos_23 pos_2 +pos_17 +pos_22 pos_3 -pos_4 -pos_5 -pos_6 pos_7 +pos_17 +pos_4 pos_8 pos_9 +pos_6 +pos_11 pos_10 -pos_1 pos_2 -pos_3 -pos_4 +pos_15 +pos_12 +pos_16 +pos_1 +pos_18 +pos_14 pos_5 -pos_6 +pos_3 +pos_13 +pos_13 +pos_11 pos_7 -pos_8 -pos_9 -pos_10 -pos_1 pos_2 -pos_3 +pos_15 +pos_1 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 pos_12 -pos_13 +pos_6 +pos_3 +pos_10 +pos_8 pos_14 -pos_15 +pos_5 pos_16 pos_1 +pos_7 +pos_8 +pos_6 pos_2 +pos_9 pos_3 pos_4 pos_5 -pos_6 +pos_1 pos_7 -pos_8 -pos_9 +pos_2 +pos_6 +pos_19 +pos_22 +pos_21 pos_10 +pos_15 +pos_12 pos_11 -pos_1 -pos_2 +pos_20 +pos_23 pos_3 -pos_4 pos_5 -pos_6 -pos_7 +pos_16 +pos_18 +pos_17 +pos_13 +pos_9 pos_8 +pos_14 +pos_4 +pos_2 +pos_6 +pos_1 +pos_4 pos_9 +pos_8 pos_10 +pos_7 +pos_3 +pos_5 pos_11 -pos_12 +pos_6 +pos_3 pos_13 +pos_10 +pos_2 +pos_8 +pos_4 +pos_12 +pos_11 pos_14 +pos_5 +pos_9 +pos_7 +pos_1 pos_15 +pos_8 +pos_6 +pos_7 +pos_4 +pos_5 +pos_3 +pos_2 +pos_1 pos_1 +pos_10 pos_2 -pos_3 -pos_4 pos_5 +pos_9 pos_6 +pos_3 pos_7 pos_8 +pos_4 +pos_5 +pos_8 +pos_3 +pos_1 +pos_6 pos_9 pos_10 +pos_4 +pos_2 +pos_7 +pos_16 pos_11 +pos_15 pos_12 -pos_13 +pos_8 pos_14 -pos_15 -pos_16 -pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_22 +pos_5 +pos_9 +pos_13 +pos_4 +pos_10 +pos_2 pos_1 +pos_7 +pos_6 +pos_3 pos_2 +pos_11 +pos_9 +pos_4 +pos_10 +pos_6 +pos_7 +pos_5 pos_3 +pos_8 +pos_1 pos_4 +pos_10 +pos_3 +pos_2 pos_5 pos_6 -pos_7 +pos_15 +pos_14 +pos_12 +pos_1 +pos_13 pos_8 pos_9 -pos_10 +pos_7 pos_11 +pos_18 +pos_21 +pos_9 pos_12 +pos_8 +pos_17 +pos_5 +pos_10 +pos_4 +pos_16 +pos_1 +pos_22 +pos_20 pos_13 pos_14 +pos_2 pos_15 +pos_3 +pos_7 +pos_6 +pos_19 +pos_11 pos_16 pos_1 -pos_2 +pos_11 +pos_15 +pos_5 +pos_10 pos_3 +pos_12 +pos_14 pos_4 -pos_5 -pos_6 -pos_7 +pos_13 pos_8 +pos_2 pos_9 -pos_10 -pos_11 -pos_12 +pos_7 +pos_6 pos_13 -pos_14 -pos_15 -pos_16 -pos_17 +pos_11 pos_1 -pos_2 -pos_3 +pos_14 pos_4 -pos_5 pos_6 +pos_3 +pos_9 +pos_2 pos_7 pos_8 -pos_9 -pos_10 -pos_11 pos_12 -pos_13 -pos_14 +pos_5 +pos_10 +pos_17 pos_15 pos_16 -pos_17 -pos_18 -pos_19 -pos_20 pos_21 -pos_22 -pos_23 +pos_19 +pos_13 +pos_16 +pos_5 +pos_14 pos_1 -pos_2 -pos_3 +pos_23 pos_4 -pos_5 +pos_11 pos_6 -pos_7 -pos_8 pos_9 +pos_2 pos_10 -pos_11 +pos_20 +pos_8 +pos_15 +pos_17 +pos_12 +pos_3 +pos_7 +pos_22 +pos_18 pos_12 +pos_9 +pos_5 pos_13 -pos_14 +pos_7 pos_15 +pos_6 +pos_3 +pos_8 +pos_14 pos_16 +pos_4 pos_1 +pos_11 +pos_10 pos_2 -pos_3 +pos_9 +pos_1 +pos_15 +pos_17 +pos_8 pos_4 -pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 +pos_22 +pos_21 +pos_20 pos_12 pos_13 -pos_14 -pos_15 +pos_19 +pos_11 pos_16 -pos_17 pos_18 -pos_19 -pos_20 -pos_21 -pos_22 -pos_1 +pos_7 +pos_5 pos_2 +pos_14 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 pos_11 +pos_6 pos_12 -pos_13 -pos_14 -pos_15 -pos_16 +pos_7 pos_17 +pos_3 +pos_4 +pos_10 pos_1 +pos_9 +pos_15 +pos_14 +pos_5 pos_2 +pos_13 +pos_16 +pos_8 +pos_12 pos_3 -pos_4 +pos_11 pos_5 pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_12 +pos_8 +pos_2 pos_13 -pos_14 pos_1 -pos_2 -pos_3 +pos_14 +pos_7 pos_4 -pos_5 +pos_9 +pos_12 pos_6 -pos_7 pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_1 +pos_9 pos_2 -pos_3 +pos_1 +pos_10 pos_4 -pos_5 -pos_6 +pos_3 pos_7 -pos_8 -pos_9 +pos_5 pos_10 -pos_11 +pos_4 pos_12 -pos_13 -pos_14 pos_1 -pos_2 +pos_8 +pos_9 pos_3 -pos_4 pos_5 +pos_13 +pos_2 +pos_11 pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_17 +pos_11 pos_18 -pos_19 +pos_6 +pos_15 pos_20 -pos_1 -pos_2 -pos_3 pos_4 +pos_16 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 +pos_13 +pos_17 +pos_3 pos_10 -pos_11 +pos_8 pos_12 -pos_13 +pos_19 pos_14 -pos_15 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_9 +pos_1 pos_7 pos_8 -pos_9 +pos_12 +pos_3 +pos_14 +pos_5 +pos_4 pos_10 +pos_9 +pos_7 pos_11 -pos_12 +pos_15 +pos_2 pos_13 +pos_1 +pos_6 pos_14 pos_15 -pos_16 -pos_17 pos_1 -pos_2 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 +pos_11 +pos_13 +pos_17 pos_8 +pos_6 pos_9 -pos_10 -pos_11 +pos_4 pos_12 +pos_7 +pos_10 +pos_5 +pos_2 +pos_16 pos_13 -pos_14 -pos_15 -pos_1 +pos_9 +pos_12 pos_2 +pos_1 +pos_8 +pos_5 +pos_15 +pos_11 pos_3 pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 +pos_6 +pos_14 pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_15 -pos_1 -pos_2 -pos_3 +pos_11 +pos_1 pos_4 -pos_5 -pos_6 +pos_3 pos_7 -pos_8 -pos_9 +pos_5 pos_10 -pos_11 -pos_12 pos_13 -pos_14 +pos_6 pos_15 +pos_2 +pos_12 +pos_8 +pos_9 +pos_22 +pos_5 pos_16 -pos_17 +pos_1 pos_18 -pos_19 pos_20 +pos_17 +pos_14 +pos_15 +pos_10 +pos_9 +pos_11 pos_21 -pos_22 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_13 pos_7 pos_8 -pos_9 -pos_1 +pos_19 +pos_12 +pos_6 +pos_4 +pos_3 pos_2 +pos_1 +pos_8 +pos_9 pos_3 -pos_4 +pos_7 pos_5 +pos_4 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 pos_13 +pos_3 +pos_2 +pos_10 pos_14 -pos_15 -pos_16 +pos_1 pos_17 -pos_18 -pos_19 -pos_20 +pos_7 pos_21 -pos_1 -pos_2 -pos_3 +pos_15 pos_4 +pos_9 +pos_20 +pos_19 +pos_8 +pos_16 pos_5 +pos_12 pos_6 -pos_7 +pos_18 +pos_11 pos_8 -pos_9 -pos_1 pos_2 -pos_3 -pos_4 +pos_1 pos_5 -pos_6 pos_7 -pos_8 pos_9 -pos_10 -pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_17 -pos_1 -pos_2 +pos_6 pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 +pos_8 +pos_4 pos_12 -pos_13 -pos_14 +pos_17 pos_15 -pos_16 +pos_11 +pos_5 +pos_14 pos_1 -pos_2 pos_3 -pos_4 -pos_5 +pos_7 +pos_13 +pos_10 pos_6 +pos_16 +pos_2 +pos_15 +pos_10 +pos_14 +pos_16 pos_7 -pos_8 +pos_6 pos_9 -pos_10 +pos_8 pos_11 +pos_2 pos_12 -pos_13 -pos_14 -pos_15 +pos_5 +pos_4 pos_1 -pos_2 +pos_13 pos_3 pos_4 -pos_5 pos_6 pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_12 -pos_13 +pos_8 +pos_5 pos_1 +pos_15 +pos_13 pos_2 +pos_14 +pos_11 +pos_9 +pos_12 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 +pos_2 pos_12 pos_13 -pos_1 -pos_2 +pos_6 pos_3 pos_4 pos_5 -pos_6 pos_7 -pos_8 -pos_9 +pos_1 pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_15 -pos_1 -pos_2 -pos_3 -pos_4 +pos_8 pos_5 -pos_6 pos_7 +pos_3 +pos_10 +pos_4 pos_8 +pos_1 pos_9 -pos_10 +pos_2 pos_11 +pos_6 pos_12 pos_13 pos_14 -pos_1 +pos_11 +pos_6 pos_2 -pos_3 pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 pos_13 -pos_14 pos_15 -pos_16 -pos_17 -pos_18 +pos_9 pos_1 -pos_2 +pos_12 +pos_8 +pos_7 pos_3 +pos_10 pos_4 pos_5 -pos_6 -pos_7 pos_8 +pos_3 +pos_2 +pos_7 +pos_1 +pos_13 +pos_6 pos_9 +pos_14 +pos_12 +pos_10 +pos_11 pos_10 pos_11 +pos_18 +pos_6 +pos_8 pos_12 pos_13 -pos_14 -pos_15 +pos_5 +pos_3 pos_16 pos_17 -pos_18 -pos_19 -pos_20 -pos_21 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 pos_7 -pos_8 pos_9 +pos_4 +pos_14 +pos_15 +pos_1 +pos_15 +pos_7 +pos_3 pos_10 +pos_21 +pos_20 +pos_2 +pos_8 +pos_16 +pos_6 +pos_5 +pos_13 +pos_18 pos_11 +pos_4 pos_12 -pos_13 +pos_17 pos_14 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 +pos_9 +pos_19 pos_6 -pos_7 -pos_8 pos_9 -pos_10 +pos_5 +pos_3 pos_11 +pos_2 +pos_10 +pos_8 +pos_7 +pos_14 pos_12 pos_13 -pos_14 -pos_15 -pos_16 -pos_17 pos_1 -pos_2 -pos_3 pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 +pos_15 +pos_14 pos_10 +pos_4 +pos_2 pos_11 +pos_8 +pos_16 +pos_5 pos_12 -pos_13 -pos_14 -pos_15 +pos_6 +pos_9 pos_1 +pos_3 +pos_17 +pos_13 pos_2 +pos_1 +pos_10 +pos_7 +pos_14 +pos_12 +pos_8 pos_3 +pos_15 +pos_6 pos_4 +pos_13 +pos_11 pos_5 +pos_9 pos_6 +pos_1 +pos_14 +pos_10 +pos_2 +pos_5 pos_7 -pos_8 pos_9 -pos_10 +pos_13 +pos_4 pos_11 +pos_8 +pos_3 pos_12 pos_13 -pos_14 -pos_1 -pos_2 +pos_8 pos_3 -pos_4 +pos_2 +pos_11 pos_5 -pos_6 pos_7 -pos_8 -pos_9 pos_10 -pos_11 +pos_4 +pos_6 pos_12 -pos_13 -pos_14 pos_15 +pos_14 +pos_9 pos_16 pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 pos_12 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 -pos_6 +pos_1 pos_7 pos_8 +pos_4 +pos_3 pos_9 -pos_10 pos_11 -pos_12 -pos_13 +pos_6 +pos_11 +pos_6 +pos_3 +pos_8 pos_14 -pos_15 pos_16 pos_17 +pos_13 +pos_15 +pos_4 +pos_12 pos_1 +pos_10 pos_2 -pos_3 -pos_4 +pos_9 pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 -pos_13 +pos_3 pos_14 -pos_15 +pos_7 pos_16 +pos_4 pos_17 -pos_18 -pos_19 pos_1 -pos_2 -pos_3 -pos_4 +pos_15 pos_5 pos_6 -pos_7 +pos_12 +pos_19 pos_8 +pos_13 +pos_11 pos_9 +pos_18 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 +pos_2 pos_15 +pos_8 pos_16 +pos_12 +pos_6 +pos_7 +pos_5 +pos_10 +pos_3 pos_1 +pos_4 pos_2 -pos_3 +pos_13 +pos_14 +pos_9 +pos_11 pos_4 pos_5 +pos_10 +pos_8 +pos_2 pos_6 +pos_11 pos_7 -pos_8 pos_9 -pos_10 -pos_11 pos_1 -pos_2 pos_3 +pos_10 +pos_13 +pos_8 +pos_18 pos_4 -pos_5 -pos_6 +pos_2 pos_7 -pos_8 pos_9 -pos_10 -pos_11 +pos_17 +pos_6 +pos_16 pos_12 -pos_13 pos_14 -pos_15 -pos_16 -pos_17 -pos_18 pos_1 -pos_2 pos_3 -pos_4 +pos_11 +pos_5 +pos_15 pos_5 -pos_6 -pos_7 pos_8 pos_9 -pos_10 -pos_11 pos_1 pos_2 +pos_11 +pos_7 +pos_10 pos_3 pos_4 -pos_5 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 +pos_5 pos_13 -pos_1 pos_2 +pos_9 pos_3 +pos_1 +pos_8 +pos_10 pos_4 -pos_5 -pos_6 +pos_12 +pos_11 pos_7 +pos_6 pos_8 +pos_3 +pos_5 +pos_13 +pos_4 +pos_11 +pos_1 pos_9 +pos_7 pos_10 -pos_11 -pos_12 -pos_13 pos_14 -pos_1 +pos_6 +pos_12 pos_2 -pos_3 pos_4 -pos_5 -pos_6 -pos_7 -pos_8 pos_9 -pos_1 -pos_2 pos_3 -pos_4 +pos_1 pos_5 +pos_2 pos_6 -pos_7 pos_8 -pos_9 -pos_10 +pos_7 pos_11 -pos_12 +pos_2 pos_13 -pos_14 +pos_4 +pos_7 pos_15 -pos_16 +pos_14 pos_1 -pos_2 -pos_3 -pos_4 -pos_5 +pos_9 pos_6 -pos_7 +pos_3 pos_8 -pos_9 pos_10 -pos_11 +pos_5 pos_12 -pos_13 -pos_14 -pos_15 +pos_16 pos_1 -pos_2 -pos_3 -pos_4 +pos_11 +pos_10 +pos_9 +pos_12 pos_5 pos_6 pos_7 +pos_3 +pos_14 +pos_13 +pos_4 +pos_15 +pos_2 pos_8 -pos_9 -pos_10 -pos_11 pos_12 -pos_13 -pos_14 +pos_11 pos_15 -pos_16 -pos_1 pos_2 -pos_3 pos_4 +pos_1 +pos_10 pos_5 pos_6 -pos_7 -pos_8 pos_9 -pos_10 -pos_11 -pos_12 +pos_8 pos_13 -pos_14 -pos_15 +pos_3 pos_16 +pos_7 +pos_14 +pos_11 +pos_7 pos_17 -pos_18 +pos_9 +pos_1 +pos_5 +pos_3 +pos_2 pos_19 +pos_18 +pos_24 +pos_6 +pos_8 +pos_16 +pos_12 pos_20 -pos_21 +pos_15 pos_22 -pos_23 -pos_24 +pos_10 pos_25 -pos_1 -pos_2 -pos_3 +pos_21 +pos_4 +pos_23 +pos_14 +pos_13 pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 -pos_1 -pos_2 pos_3 -pos_4 +pos_1 +pos_9 +pos_8 pos_5 +pos_2 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 pos_11 +pos_2 pos_12 pos_13 +pos_10 +pos_9 pos_1 -pos_2 -pos_3 +pos_6 pos_4 +pos_8 +pos_7 pos_5 -pos_6 +pos_3 +pos_15 pos_7 -pos_8 -pos_9 -pos_10 pos_11 -pos_12 pos_13 -pos_14 -pos_15 +pos_19 +pos_2 +pos_6 +pos_8 +pos_12 pos_16 +pos_22 pos_17 pos_18 -pos_19 -pos_20 -pos_21 -pos_22 +pos_4 pos_1 -pos_2 +pos_14 +pos_20 pos_3 -pos_4 +pos_21 +pos_9 +pos_10 pos_5 -pos_6 -pos_7 +pos_13 pos_8 pos_9 +pos_6 +pos_2 pos_10 -pos_11 +pos_7 pos_12 -pos_13 -pos_14 pos_15 pos_16 +pos_11 pos_1 -pos_2 -pos_3 +pos_5 +pos_14 pos_4 +pos_3 pos_5 -pos_6 -pos_7 +pos_3 +pos_14 +pos_18 +pos_10 +pos_13 pos_8 +pos_15 +pos_1 pos_9 -pos_10 pos_11 +pos_7 pos_12 -pos_13 -pos_14 -pos_15 pos_16 +pos_4 +pos_6 +pos_2 pos_17 -pos_18 -pos_1 pos_2 -pos_3 -pos_4 -pos_5 +pos_16 pos_6 -pos_7 -pos_8 -pos_9 -pos_10 -pos_11 -pos_12 +pos_20 pos_13 +pos_12 pos_14 -pos_15 -pos_16 pos_17 -pos_18 +pos_5 +pos_10 pos_19 -pos_20 -pos_1 -pos_2 +pos_18 pos_3 +pos_8 +pos_1 pos_4 -pos_5 -pos_6 pos_7 -pos_8 +pos_15 pos_9 -pos_10 pos_11 -pos_12 -pos_13 +pos_8 pos_14 pos_1 -pos_2 +pos_6 +pos_10 +pos_5 pos_3 +pos_13 +pos_12 pos_4 -pos_5 -pos_6 +pos_2 +pos_11 +pos_9 pos_7 pos_8 -pos_9 -pos_10 pos_11 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 -pos_6 pos_7 -pos_8 -pos_9 +pos_3 pos_1 +pos_5 +pos_9 +pos_6 +pos_10 pos_2 pos_3 +pos_6 +pos_1 pos_4 +pos_2 pos_5 -pos_6 +pos_9 pos_7 pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 +pos_2 pos_15 +pos_10 +pos_8 +pos_9 +pos_5 pos_16 -pos_1 -pos_2 -pos_3 pos_4 -pos_5 +pos_13 +pos_3 +pos_7 +pos_1 pos_6 +pos_12 +pos_14 +pos_15 pos_7 -pos_8 pos_9 -pos_10 +pos_6 +pos_14 +pos_19 +pos_8 pos_11 -pos_12 +pos_18 +pos_5 +pos_10 pos_13 -pos_14 -pos_15 -pos_16 pos_17 -pos_18 -pos_19 pos_1 +pos_12 pos_2 pos_3 pos_4 -pos_5 -pos_6 -pos_7 +pos_16 +pos_13 +pos_2 +pos_17 pos_8 -pos_9 -pos_10 +pos_16 pos_11 -pos_12 -pos_13 -pos_14 +pos_1 pos_15 -pos_16 -pos_17 +pos_4 +pos_10 +pos_9 +pos_14 pos_18 -pos_19 -pos_1 -pos_2 pos_3 -pos_4 pos_5 pos_6 pos_7 -pos_8 -pos_9 +pos_12 +pos_19 +pos_5 +pos_7 +pos_3 +pos_1 pos_10 +pos_6 pos_11 -pos_1 pos_2 -pos_3 +pos_9 +pos_8 pos_4 -pos_5 +pos_9 pos_6 -pos_7 pos_8 -pos_9 -pos_10 -pos_11 -pos_1 -pos_2 -pos_3 pos_4 pos_5 -pos_6 +pos_3 +pos_11 pos_7 -pos_8 -pos_9 +pos_2 pos_10 -pos_11 -pos_12 -pos_13 pos_1 -pos_2 pos_3 +pos_1 +pos_12 +pos_9 +pos_2 +pos_13 +pos_7 +pos_10 pos_4 -pos_5 pos_6 -pos_7 pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_13 -pos_14 -pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 +pos_12 pos_7 -pos_8 -pos_9 pos_10 +pos_4 +pos_2 +pos_1 pos_11 -pos_12 -pos_13 +pos_9 pos_14 -pos_1 -pos_2 +pos_13 +pos_6 pos_3 -pos_4 pos_5 +pos_8 +pos_8 pos_6 +pos_14 +pos_5 +pos_4 pos_7 -pos_8 pos_9 -pos_10 -pos_11 pos_12 pos_13 +pos_10 +pos_3 pos_1 pos_2 -pos_3 +pos_11 +pos_1 +pos_2 +pos_6 pos_4 +pos_10 +pos_11 pos_5 -pos_6 +pos_13 +pos_3 +pos_12 pos_7 -pos_8 pos_9 +pos_8 +pos_3 +pos_1 pos_10 +pos_5 +pos_8 +pos_2 +pos_4 +pos_6 +pos_15 pos_11 -pos_12 pos_13 -pos_14 -pos_15 +pos_9 +pos_7 pos_16 -pos_1 -pos_2 -pos_3 +pos_12 +pos_14 pos_4 pos_5 +pos_2 pos_6 +pos_3 pos_1 +pos_4 +pos_16 +pos_20 +pos_11 pos_2 +pos_12 pos_3 -pos_4 -pos_5 -pos_6 -pos_7 +pos_19 +pos_21 pos_8 -pos_9 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 pos_15 -pos_16 -pos_17 +pos_13 pos_18 -pos_19 -pos_20 -pos_21 +pos_6 +pos_17 +pos_7 pos_1 -pos_2 -pos_3 -pos_4 pos_5 -pos_6 +pos_14 +pos_9 +pos_14 +pos_11 pos_7 pos_8 -pos_9 pos_10 -pos_11 pos_12 +pos_2 +pos_6 +pos_5 +pos_4 +pos_9 pos_13 -pos_14 -pos_15 +pos_3 pos_16 pos_1 -pos_2 -pos_3 -pos_4 +pos_15 +pos_1 pos_5 -pos_6 -pos_7 -pos_8 -pos_9 -pos_10 pos_11 -pos_12 -pos_1 +pos_9 pos_2 pos_3 -pos_4 -pos_5 pos_6 +pos_4 pos_7 -pos_8 -pos_9 pos_10 -pos_11 +pos_8 pos_12 -pos_13 -pos_14 -pos_15 -pos_16 -pos_1 -pos_2 pos_3 +pos_8 pos_4 pos_5 +pos_16 +pos_1 +pos_15 +pos_11 pos_6 -pos_7 -pos_8 -pos_9 pos_10 -pos_11 -pos_1 pos_2 +pos_14 +pos_9 +pos_13 +pos_12 +pos_7 +pos_9 pos_3 +pos_11 pos_4 -pos_5 +pos_8 +pos_1 pos_6 pos_7 -pos_8 -pos_9 +pos_2 pos_10 -pos_11 -pos_12 -pos_13 -pos_14 +pos_5 +pos_21 +pos_7 +pos_23 +pos_2 +pos_24 pos_15 -pos_16 -pos_17 -pos_18 pos_19 +pos_13 +pos_8 +pos_16 +pos_12 +pos_4 +pos_11 +pos_9 pos_20 -pos_21 pos_22 -pos_23 -pos_24 +pos_18 +pos_5 +pos_10 +pos_6 pos_1 -pos_2 +pos_17 pos_3 -pos_4 -pos_5 +pos_14 +pos_8 pos_6 +pos_1 +pos_11 +pos_5 pos_7 -pos_8 +pos_4 +pos_3 pos_9 pos_10 -pos_11 -pos_12 -pos_1 pos_2 +pos_12 +pos_7 pos_3 -pos_4 pos_5 +pos_4 +pos_10 pos_6 -pos_7 pos_8 +pos_2 pos_9 -pos_10 +pos_1 From 4ecbbf9765bec2dc8cc1950c286c5d32b94e7b2c Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:05:28 -0700 Subject: [PATCH 27/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index abada79cb2c1..20a06c3db662 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -756,16 +756,28 @@ def test_ranking_with_position_information(tmp_path): } copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) + copyfile(str(rank_example_dir / 'rank.test'), str(tmp_path / 'rank.test')) + copyfile(str(rank_example_dir / 'rank.test.query'), str(tmp_path / 'rank.test.query')) + + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') + copyfile(str(rank_example_dir / '_rank.train.position'), str(tmp_path / 'rank.train.position')) lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) - gbm = lgb.train(params, lgb_train, num_boost_round=50) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') + + assert gbm_baseline.eval_valid()['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.eval_valid()['valid_0']['ndcg@3'], 0.02) + # add extra row to position file with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') file.close() lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): - lgb.train(params, lgb_train, num_boost_round=50) + lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From 3bc941559f35dc16f9e82868fbfae6aa87fb01b5 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 22 Jun 2023 00:08:16 -0700 Subject: [PATCH 28/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 20a06c3db662..0697e423f42f 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -752,7 +752,9 @@ def test_ranking_with_position_information(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' params = { 'objective': 'lambdarank', - 'verbose': -1 + 'verbose': -1, + 'eval_at': 3, + 'eval_metric': 'ndcg' } copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) @@ -761,12 +763,12 @@ def test_ranking_with_position_information(tmp_path): lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] - gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') + gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) copyfile(str(rank_example_dir / '_rank.train.position'), str(tmp_path / 'rank.train.position')) lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] - gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') + gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) assert gbm_baseline.eval_valid()['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.eval_valid()['valid_0']['ndcg@3'], 0.02) @@ -777,7 +779,7 @@ def test_ranking_with_position_information(tmp_path): lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): - lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50, eval_at=[3], eval_metric='ndcg') + lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From 7df432387fad98cb0ee5055c3264f5f43e584e40 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 22 Jun 2023 00:45:12 -0700 Subject: [PATCH 29/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 0697e423f42f..ec0a4be0deab 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -753,7 +753,7 @@ def test_ranking_with_position_information(tmp_path): params = { 'objective': 'lambdarank', 'verbose': -1, - 'eval_at': 3, + 'eval_at': [3], 'eval_metric': 'ndcg' } copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) From 844d1012b9fbdeb05a6ca37b61e224402001f663 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 22 Jun 2023 01:48:55 -0700 Subject: [PATCH 30/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index ec0a4be0deab..4583e440df21 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -770,7 +770,7 @@ def test_ranking_with_position_information(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - assert gbm_baseline.eval_valid()['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.eval_valid()['valid_0']['ndcg@3'], 0.02) + assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) # add extra row to position file with open(str(tmp_path / 'rank.train.position'), 'a') as file: From 1ae780ae15741d5069316e87dc7cefe8cd57b679 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:25:22 -0700 Subject: [PATCH 31/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 4583e440df21..62af9144f5d3 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -770,6 +770,7 @@ def test_ranking_with_position_information(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) # add extra row to position file From dffdda16a19ebdd837abc42f87d7f9db39984294 Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Mon, 26 Jun 2023 16:01:27 +0800 Subject: [PATCH 32/97] Update the position of import statement --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 62af9144f5d3..c71b942c4778 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -9,6 +9,7 @@ import re from os import getenv from pathlib import Path +from shutil import copyfile import numpy as np import psutil @@ -25,7 +26,6 @@ make_synthetic_regression, mse_obj, pickle_and_unpickle_object, sklearn_multiclass_custom_objective, softmax) -from shutil import copyfile decreasing_generator = itertools.count(0, -1) From 055bd1ced7f48780a281fadca98a986f331aa075 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:51:25 -0700 Subject: [PATCH 33/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 47 ++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 3f76042b04c8..58db7da2350b 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -25,7 +25,10 @@ namespace LightGBM { class RankingObjective : public ObjectiveFunction { public: explicit RankingObjective(const Config& config) - : seed_(config.objective_seed) {} + : seed_(config.objective_seed) { + learning_rate_ = config.learning_rate; + position_bias_regularizer_ = config.lambdarank_position_bias_regularizer; + } explicit RankingObjective(const std::vector&) : seed_(0) {} @@ -49,6 +52,8 @@ class RankingObjective : public ObjectiveFunction { Log::Fatal("Ranking tasks require query information"); } num_queries_ = metadata.num_queries(); + // initialize position bias vectors + pos_biases_.resize(num_position_ids_, 0.0); } void GetGradients(const double* score, score_t* gradients, @@ -57,7 +62,13 @@ class RankingObjective : public ObjectiveFunction { for (data_size_t i = 0; i < num_queries_; ++i) { const data_size_t start = query_boundaries_[i]; const data_size_t cnt = query_boundaries_[i + 1] - query_boundaries_[i]; - GetGradientsForOneQuery(i, cnt, label_ + start, score + start, + std::vector score_adjusted; + if (num_position_ids_ > 0) { + for (data_size_t j = 0; j < cnt; ++j) { + score_adjusted.push_back(score[start + j] + pos_biases_[positions_[start + j]]); + } + } + GetGradientsForOneQuery(i, cnt, label_ + start, num_position_ids_ > 0? score_adjusted.data(): score + start, gradients + start, hessians + start); if (weights_ != nullptr) { for (data_size_t j = 0; j < cnt; ++j) { @@ -107,6 +118,12 @@ class RankingObjective : public ObjectiveFunction { data_size_t num_position_ids_; /*! \brief Query boundaries */ const data_size_t* query_boundaries_; + /*! \brief Position bias factors */ + mutable std::vector pos_biases_; + /*! \brief Learning rate to update position bias factors */ + double learning_rate_; + /*! \brief Position bias regularizer */ + double position_bias_regularizer_; }; /*! @@ -128,7 +145,6 @@ class LambdarankNDCG : public RankingObjective { if (sigmoid_ <= 0.0) { Log::Fatal("Sigmoid param %f should be greater than zero", sigmoid_); } - learning_rate_ = config.learning_rate; } explicit LambdarankNDCG(const std::vector& strs) @@ -153,8 +169,6 @@ class LambdarankNDCG : public RankingObjective { } // construct Sigmoid table to speed up Sigmoid transform ConstructSigmoidTable(); - // initialize position bias vectors - pos_biases_.resize(num_position_ids_, 0.0); } inline void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt, @@ -174,8 +188,8 @@ class LambdarankNDCG : public RankingObjective { sorted_idx[i] = i; } std::stable_sort( - sorted_idx.begin(), sorted_idx.end(), - [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); + sorted_idx.begin(), sorted_idx.end(), + [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); // get best and worst score const double best_score = score[sorted_idx[0]]; data_size_t worst_idx = cnt - 1; @@ -201,18 +215,14 @@ class LambdarankNDCG : public RankingObjective { } const data_size_t high = sorted_idx[high_rank]; const int high_label = static_cast(label[high]); - double high_score = score[high]; + const double high_score = score[high]; const double high_label_gain = label_gain_[high_label]; const double high_discount = DCGCalculator::GetDiscount(high_rank); const data_size_t low = sorted_idx[low_rank]; const int low_label = static_cast(label[low]); - double low_score = score[low]; + const double low_score = score[low]; const double low_label_gain = label_gain_[low_label]; const double low_discount = DCGCalculator::GetDiscount(low_rank); - if (num_position_ids_ > 0) { - high_score += pos_biases_[positions_[query_boundaries_[query_id] + high]]; - low_score += pos_biases_[positions_[query_boundaries_[query_id] + low]]; - } const double delta_score = high_score - low_score; @@ -288,6 +298,7 @@ class LambdarankNDCG : public RankingObjective { // create per-thread buffers for first and second derivatives of utility w.r.t. position bias factors std::vector bias_first_derivatives(num_position_ids_ * num_threads, 0.0); std::vector bias_second_derivatives(num_position_ids_ * num_threads, 0.0); + std::vector instance_counts(num_position_ids_ * num_threads, 0); #pragma omp parallel for schedule(guided) for (data_size_t i = 0; i < num_data_; i++) { // get thread ID @@ -297,17 +308,23 @@ class LambdarankNDCG : public RankingObjective { bias_first_derivatives[offset] -= lambdas[i]; // accumulate second derivatives of utility w.r.t. position bias factors, for each position bias_second_derivatives[offset] -= hessians[i]; + instance_counts[offset]++; } #pragma omp parallel for schedule(guided) for (data_size_t i = 0; i < num_position_ids_; i++) { double bias_first_derivative = 0.0; double bias_second_derivative = 0.0; + int instance_count = 0; // aggregate derivatives from per-thread buffers for (int tid = 0; tid < num_threads; tid++) { size_t offset = static_cast(i + tid * num_position_ids_); bias_first_derivative += bias_first_derivatives[offset]; bias_second_derivative += bias_second_derivatives[offset]; + instance_count += instance_counts[offset]; } + // L2 regularization on position bias factors + bias_first_derivative -= pos_biases_[i] * position_bias_regularizer_ * instance_count; + bias_second_derivative -= position_bias_regularizer_ * instance_count; // do Newton-Rhapson step to update position bias factors pos_biases_[i] += learning_rate_ * bias_first_derivative / (std::abs(bias_second_derivative) + 0.001); } @@ -351,10 +368,6 @@ class LambdarankNDCG : public RankingObjective { double max_sigmoid_input_ = 50; /*! \brief Factor that covert score to bin in Sigmoid table */ double sigmoid_table_idx_factor_; - /*! \brief Position bias factors */ - mutable std::vector pos_biases_; - /*! \brief Learning rate to update position bias factors */ - double learning_rate_; }; /*! From 2885928c890c576480da51d2ac70bc4b1b1f6a43 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:53:15 -0700 Subject: [PATCH 34/97] Update config.h --- include/LightGBM/config.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/LightGBM/config.h b/include/LightGBM/config.h index 89318a7af246..36d47715aa5c 100644 --- a/include/LightGBM/config.h +++ b/include/LightGBM/config.h @@ -959,6 +959,10 @@ struct Config { // desc = separate by ``,`` std::vector label_gain; + // check = >=0.0 + // desc = used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. + double lambdarank_position_bias_regularizer = 0.0; + #ifndef __NVCC__ #pragma endregion From a56687235106b717697311a112b9afd420937caf Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:53:44 -0700 Subject: [PATCH 35/97] Update config_auto.cpp --- src/io/config_auto.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp index 0906ba4b6439..c495dd053b8b 100644 --- a/src/io/config_auto.cpp +++ b/src/io/config_auto.cpp @@ -304,6 +304,7 @@ const std::unordered_set& Config::parameter_set() { "lambdarank_truncation_level", "lambdarank_norm", "label_gain", + "lambdarank_position_bias_regularizer", "metric", "metric_freq", "is_provide_training_metric", @@ -619,6 +620,9 @@ void Config::GetMembersFromString(const std::unordered_map(tmp_str, ','); } + GetDouble(params, "lambdarank_position_bias_regularizer", &lambdarank_position_bias_regularizer); + CHECK_GE(lambdarank_position_bias_regularizer, 0.0); + GetInt(params, "metric_freq", &metric_freq); CHECK_GT(metric_freq, 0); @@ -754,6 +758,7 @@ std::string Config::SaveMembersToString() const { str_buf << "[lambdarank_truncation_level: " << lambdarank_truncation_level << "]\n"; str_buf << "[lambdarank_norm: " << lambdarank_norm << "]\n"; str_buf << "[label_gain: " << Common::Join(label_gain, ",") << "]\n"; + str_buf << "[lambdarank_position_bias_regularizer: " << lambdarank_position_bias_regularizer << "]\n"; str_buf << "[eval_at: " << Common::Join(eval_at, ",") << "]\n"; str_buf << "[multi_error_top_k: " << multi_error_top_k << "]\n"; str_buf << "[auc_mu_weights: " << Common::Join(auc_mu_weights, ",") << "]\n"; @@ -893,6 +898,7 @@ const std::unordered_map>& Config::paramet {"lambdarank_truncation_level", {}}, {"lambdarank_norm", {}}, {"label_gain", {}}, + {"lambdarank_position_bias_regularizer", {} }, {"metric", {"metrics", "metric_types"}}, {"metric_freq", {"output_freq"}}, {"is_provide_training_metric", {"training_metric", "is_training_metric", "train_metric"}}, @@ -1035,6 +1041,7 @@ const std::unordered_map& Config::ParameterTypes() { {"lambdarank_truncation_level", "int"}, {"lambdarank_norm", "bool"}, {"label_gain", "vector"}, + {"lambdarank_position_bias_regularizer", "double" }, {"metric", "vector"}, {"metric_freq", "int"}, {"is_provide_training_metric", "bool"}, From 0707cc00edb1c7a2542e037998c41c0e560a72cb Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 27 Jun 2023 02:59:07 -0700 Subject: [PATCH 36/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 58db7da2350b..cae86b9bea86 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -188,8 +188,8 @@ class LambdarankNDCG : public RankingObjective { sorted_idx[i] = i; } std::stable_sort( - sorted_idx.begin(), sorted_idx.end(), - [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); + sorted_idx.begin(), sorted_idx.end(), + [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); // get best and worst score const double best_score = score[sorted_idx[0]]; data_size_t worst_idx = cnt - 1; From 64ec0984167f0cb474837e0fb99f34fdbccff314 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 27 Jun 2023 03:01:09 -0700 Subject: [PATCH 37/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index cae86b9bea86..eb9184626373 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -189,7 +189,7 @@ class LambdarankNDCG : public RankingObjective { } std::stable_sort( sorted_idx.begin(), sorted_idx.end(), - [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); + [score](data_size_t a, data_size_t b) { return score[a] > score[b]; }); // get best and worst score const double best_score = score[sorted_idx[0]]; data_size_t worst_idx = cnt - 1; From dd58f6901ad60050682b827b80c20be6fd46ceb6 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Thu, 29 Jun 2023 11:26:32 +0000 Subject: [PATCH 38/97] update documentation --- docs/Parameters.rst | 4 ++++ src/io/config_auto.cpp | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/Parameters.rst b/docs/Parameters.rst index aee1cc4e7f84..072aa994d1e6 100644 --- a/docs/Parameters.rst +++ b/docs/Parameters.rst @@ -1125,6 +1125,10 @@ Objective Parameters - separate by ``,`` +- ``lambdarank_position_bias_regularizer`` :raw-html:`🔗︎`, default = ``0.0``, type = double, constraints: ``lambdarank_position_bias_regularizer >= 0.0`` + + - used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. + Metric Parameters ----------------- diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp index c495dd053b8b..746ddafa46b9 100644 --- a/src/io/config_auto.cpp +++ b/src/io/config_auto.cpp @@ -898,7 +898,7 @@ const std::unordered_map>& Config::paramet {"lambdarank_truncation_level", {}}, {"lambdarank_norm", {}}, {"label_gain", {}}, - {"lambdarank_position_bias_regularizer", {} }, + {"lambdarank_position_bias_regularizer", {}}, {"metric", {"metrics", "metric_types"}}, {"metric_freq", {"output_freq"}}, {"is_provide_training_metric", {"training_metric", "is_training_metric", "train_metric"}}, @@ -1041,7 +1041,7 @@ const std::unordered_map& Config::ParameterTypes() { {"lambdarank_truncation_level", "int"}, {"lambdarank_norm", "bool"}, {"label_gain", "vector"}, - {"lambdarank_position_bias_regularizer", "double" }, + {"lambdarank_position_bias_regularizer", "double"}, {"metric", "vector"}, {"metric_freq", "int"}, {"is_provide_training_metric", "bool"}, From 58666e36caa535626c2126b70c65612ad30d3034 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Thu, 29 Jun 2023 11:37:30 +0000 Subject: [PATCH 39/97] remove extra blank line --- tests/python_package_test/test_engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index c71b942c4778..fc7b702405c9 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -26,7 +26,6 @@ make_synthetic_regression, mse_obj, pickle_and_unpickle_object, sklearn_multiclass_custom_objective, softmax) - decreasing_generator = itertools.count(0, -1) From 04e66edab71430693d98d0454a590c88692b3771 Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Wed, 5 Jul 2023 22:01:50 +0800 Subject: [PATCH 40/97] Update src/io/metadata.cpp Co-authored-by: James Lamb --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index c29299c3553a..98d05fbe4e50 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -266,7 +266,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector 0 && num_positions_ != num_all_data) { positions_.clear(); num_positions_ = 0; - Log::Fatal("Positions size doesn't match data size"); + Log::Fatal("Positions size (%i) doesn't match data size (%i)", num_positions_, num_data_); } // get local positions if (!positions_.empty()) { From a8c77c07faed54e85eadfe5c27196fd7650c75e9 Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Wed, 5 Jul 2023 22:02:03 +0800 Subject: [PATCH 41/97] Update src/io/metadata.cpp Co-authored-by: James Lamb --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 98d05fbe4e50..424081aee344 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -221,7 +221,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector Date: Mon, 10 Jul 2023 03:27:52 +0000 Subject: [PATCH 42/97] remove _rank.train.position --- examples/lambdarank/_rank.train.position | 3005 ---------------------- src/io/metadata.cpp | 2 +- src/objective/rank_objective.hpp | 2 +- tests/python_package_test/test_engine.py | 9 +- 4 files changed, 9 insertions(+), 3009 deletions(-) delete mode 100644 examples/lambdarank/_rank.train.position diff --git a/examples/lambdarank/_rank.train.position b/examples/lambdarank/_rank.train.position deleted file mode 100644 index 7aa6dcd9797e..000000000000 --- a/examples/lambdarank/_rank.train.position +++ /dev/null @@ -1,3005 +0,0 @@ -pos_1 -pos_8 -pos_4 -pos_1 -pos_2 -pos_12 -pos_10 -pos_7 -pos_3 -pos_11 -pos_13 -pos_9 -pos_6 -pos_5 -pos_1 -pos_5 -pos_3 -pos_4 -pos_2 -pos_1 -pos_7 -pos_2 -pos_5 -pos_8 -pos_4 -pos_6 -pos_3 -pos_6 -pos_16 -pos_1 -pos_7 -pos_14 -pos_19 -pos_4 -pos_5 -pos_12 -pos_2 -pos_17 -pos_3 -pos_8 -pos_15 -pos_18 -pos_11 -pos_9 -pos_10 -pos_13 -pos_7 -pos_6 -pos_8 -pos_2 -pos_9 -pos_11 -pos_4 -pos_12 -pos_1 -pos_5 -pos_3 -pos_10 -pos_3 -pos_12 -pos_11 -pos_9 -pos_17 -pos_1 -pos_2 -pos_8 -pos_4 -pos_18 -pos_16 -pos_7 -pos_15 -pos_13 -pos_5 -pos_6 -pos_14 -pos_10 -pos_3 -pos_2 -pos_1 -pos_4 -pos_5 -pos_13 -pos_5 -pos_10 -pos_11 -pos_7 -pos_4 -pos_12 -pos_6 -pos_8 -pos_14 -pos_1 -pos_2 -pos_3 -pos_9 -pos_5 -pos_13 -pos_8 -pos_2 -pos_11 -pos_7 -pos_9 -pos_10 -pos_12 -pos_6 -pos_3 -pos_1 -pos_4 -pos_5 -pos_4 -pos_2 -pos_7 -pos_1 -pos_8 -pos_6 -pos_3 -pos_5 -pos_4 -pos_1 -pos_6 -pos_3 -pos_2 -pos_8 -pos_9 -pos_7 -pos_14 -pos_4 -pos_9 -pos_12 -pos_7 -pos_16 -pos_1 -pos_10 -pos_11 -pos_13 -pos_2 -pos_15 -pos_3 -pos_8 -pos_6 -pos_5 -pos_4 -pos_1 -pos_10 -pos_5 -pos_9 -pos_3 -pos_11 -pos_7 -pos_8 -pos_2 -pos_6 -pos_19 -pos_10 -pos_3 -pos_1 -pos_17 -pos_9 -pos_18 -pos_2 -pos_5 -pos_15 -pos_8 -pos_11 -pos_21 -pos_12 -pos_14 -pos_16 -pos_4 -pos_6 -pos_7 -pos_13 -pos_20 -pos_2 -pos_3 -pos_7 -pos_9 -pos_11 -pos_4 -pos_13 -pos_10 -pos_8 -pos_12 -pos_5 -pos_1 -pos_14 -pos_6 -pos_20 -pos_5 -pos_7 -pos_4 -pos_15 -pos_9 -pos_6 -pos_11 -pos_16 -pos_1 -pos_8 -pos_13 -pos_17 -pos_21 -pos_19 -pos_14 -pos_2 -pos_18 -pos_3 -pos_12 -pos_10 -pos_4 -pos_1 -pos_9 -pos_6 -pos_8 -pos_7 -pos_5 -pos_2 -pos_3 -pos_12 -pos_5 -pos_10 -pos_11 -pos_3 -pos_8 -pos_4 -pos_6 -pos_1 -pos_2 -pos_14 -pos_7 -pos_9 -pos_13 -pos_11 -pos_6 -pos_5 -pos_9 -pos_4 -pos_10 -pos_2 -pos_3 -pos_1 -pos_7 -pos_8 -pos_14 -pos_7 -pos_17 -pos_18 -pos_1 -pos_8 -pos_4 -pos_10 -pos_20 -pos_3 -pos_12 -pos_2 -pos_15 -pos_16 -pos_19 -pos_11 -pos_13 -pos_9 -pos_5 -pos_6 -pos_7 -pos_9 -pos_4 -pos_13 -pos_10 -pos_12 -pos_6 -pos_17 -pos_1 -pos_3 -pos_18 -pos_16 -pos_11 -pos_14 -pos_2 -pos_8 -pos_15 -pos_5 -pos_9 -pos_12 -pos_11 -pos_8 -pos_10 -pos_13 -pos_2 -pos_1 -pos_6 -pos_7 -pos_5 -pos_4 -pos_3 -pos_19 -pos_13 -pos_12 -pos_17 -pos_14 -pos_6 -pos_8 -pos_9 -pos_4 -pos_10 -pos_1 -pos_11 -pos_2 -pos_7 -pos_20 -pos_16 -pos_18 -pos_3 -pos_5 -pos_15 -pos_3 -pos_6 -pos_14 -pos_20 -pos_1 -pos_2 -pos_8 -pos_15 -pos_4 -pos_12 -pos_18 -pos_5 -pos_10 -pos_17 -pos_16 -pos_13 -pos_22 -pos_7 -pos_19 -pos_11 -pos_21 -pos_9 -pos_11 -pos_15 -pos_10 -pos_9 -pos_14 -pos_13 -pos_6 -pos_19 -pos_5 -pos_16 -pos_3 -pos_21 -pos_12 -pos_4 -pos_2 -pos_17 -pos_18 -pos_22 -pos_1 -pos_20 -pos_7 -pos_8 -pos_9 -pos_4 -pos_6 -pos_7 -pos_12 -pos_3 -pos_2 -pos_8 -pos_13 -pos_5 -pos_10 -pos_11 -pos_1 -pos_15 -pos_9 -pos_7 -pos_8 -pos_3 -pos_14 -pos_5 -pos_12 -pos_10 -pos_1 -pos_16 -pos_6 -pos_2 -pos_4 -pos_17 -pos_11 -pos_13 -pos_3 -pos_2 -pos_8 -pos_10 -pos_4 -pos_7 -pos_6 -pos_9 -pos_5 -pos_1 -pos_10 -pos_7 -pos_4 -pos_6 -pos_2 -pos_13 -pos_3 -pos_8 -pos_1 -pos_9 -pos_11 -pos_12 -pos_5 -pos_7 -pos_4 -pos_11 -pos_12 -pos_10 -pos_6 -pos_5 -pos_9 -pos_3 -pos_1 -pos_2 -pos_8 -pos_1 -pos_8 -pos_3 -pos_6 -pos_9 -pos_10 -pos_7 -pos_4 -pos_5 -pos_11 -pos_13 -pos_2 -pos_12 -pos_2 -pos_9 -pos_10 -pos_6 -pos_3 -pos_11 -pos_4 -pos_8 -pos_13 -pos_1 -pos_12 -pos_7 -pos_5 -pos_6 -pos_23 -pos_7 -pos_19 -pos_14 -pos_2 -pos_4 -pos_16 -pos_10 -pos_8 -pos_3 -pos_1 -pos_18 -pos_15 -pos_12 -pos_17 -pos_22 -pos_20 -pos_21 -pos_13 -pos_5 -pos_9 -pos_11 -pos_4 -pos_8 -pos_11 -pos_18 -pos_6 -pos_1 -pos_2 -pos_10 -pos_7 -pos_5 -pos_3 -pos_12 -pos_16 -pos_15 -pos_17 -pos_9 -pos_13 -pos_14 -pos_2 -pos_3 -pos_5 -pos_11 -pos_10 -pos_6 -pos_4 -pos_8 -pos_13 -pos_1 -pos_12 -pos_9 -pos_7 -pos_17 -pos_10 -pos_14 -pos_4 -pos_12 -pos_19 -pos_3 -pos_9 -pos_6 -pos_1 -pos_13 -pos_7 -pos_11 -pos_8 -pos_15 -pos_16 -pos_18 -pos_2 -pos_20 -pos_5 -pos_3 -pos_5 -pos_7 -pos_9 -pos_12 -pos_6 -pos_2 -pos_4 -pos_11 -pos_10 -pos_8 -pos_1 -pos_3 -pos_7 -pos_4 -pos_9 -pos_2 -pos_15 -pos_5 -pos_13 -pos_16 -pos_8 -pos_17 -pos_14 -pos_10 -pos_21 -pos_6 -pos_18 -pos_1 -pos_12 -pos_11 -pos_19 -pos_20 -pos_22 -pos_3 -pos_12 -pos_1 -pos_6 -pos_7 -pos_4 -pos_11 -pos_14 -pos_10 -pos_2 -pos_8 -pos_5 -pos_9 -pos_13 -pos_13 -pos_7 -pos_10 -pos_3 -pos_11 -pos_6 -pos_1 -pos_4 -pos_5 -pos_9 -pos_8 -pos_2 -pos_12 -pos_2 -pos_15 -pos_18 -pos_6 -pos_20 -pos_12 -pos_23 -pos_4 -pos_9 -pos_11 -pos_10 -pos_5 -pos_1 -pos_3 -pos_14 -pos_13 -pos_7 -pos_17 -pos_19 -pos_22 -pos_21 -pos_16 -pos_8 -pos_9 -pos_4 -pos_1 -pos_8 -pos_13 -pos_3 -pos_10 -pos_12 -pos_7 -pos_6 -pos_2 -pos_5 -pos_11 -pos_5 -pos_6 -pos_3 -pos_4 -pos_11 -pos_2 -pos_8 -pos_13 -pos_7 -pos_10 -pos_14 -pos_12 -pos_9 -pos_1 -pos_8 -pos_12 -pos_2 -pos_3 -pos_4 -pos_10 -pos_13 -pos_11 -pos_14 -pos_1 -pos_5 -pos_9 -pos_6 -pos_7 -pos_2 -pos_1 -pos_4 -pos_5 -pos_3 -pos_3 -pos_4 -pos_11 -pos_8 -pos_1 -pos_13 -pos_5 -pos_10 -pos_12 -pos_6 -pos_7 -pos_2 -pos_9 -pos_6 -pos_2 -pos_7 -pos_11 -pos_13 -pos_5 -pos_9 -pos_8 -pos_15 -pos_12 -pos_1 -pos_10 -pos_14 -pos_4 -pos_3 -pos_2 -pos_5 -pos_11 -pos_4 -pos_14 -pos_3 -pos_6 -pos_13 -pos_10 -pos_8 -pos_7 -pos_12 -pos_1 -pos_9 -pos_12 -pos_14 -pos_6 -pos_8 -pos_9 -pos_7 -pos_3 -pos_10 -pos_1 -pos_5 -pos_2 -pos_13 -pos_11 -pos_4 -pos_5 -pos_9 -pos_3 -pos_7 -pos_6 -pos_12 -pos_15 -pos_16 -pos_2 -pos_4 -pos_10 -pos_1 -pos_13 -pos_14 -pos_8 -pos_11 -pos_4 -pos_12 -pos_8 -pos_13 -pos_2 -pos_9 -pos_14 -pos_7 -pos_16 -pos_11 -pos_10 -pos_15 -pos_1 -pos_6 -pos_3 -pos_5 -pos_13 -pos_14 -pos_6 -pos_4 -pos_7 -pos_9 -pos_11 -pos_10 -pos_1 -pos_2 -pos_3 -pos_15 -pos_8 -pos_12 -pos_5 -pos_9 -pos_11 -pos_19 -pos_18 -pos_13 -pos_14 -pos_6 -pos_7 -pos_5 -pos_4 -pos_12 -pos_1 -pos_17 -pos_20 -pos_10 -pos_15 -pos_3 -pos_8 -pos_2 -pos_21 -pos_16 -pos_4 -pos_8 -pos_6 -pos_12 -pos_7 -pos_10 -pos_1 -pos_17 -pos_20 -pos_13 -pos_16 -pos_3 -pos_21 -pos_19 -pos_5 -pos_18 -pos_22 -pos_15 -pos_14 -pos_11 -pos_9 -pos_2 -pos_3 -pos_1 -pos_6 -pos_8 -pos_2 -pos_5 -pos_7 -pos_9 -pos_4 -pos_10 -pos_10 -pos_15 -pos_17 -pos_20 -pos_4 -pos_13 -pos_5 -pos_2 -pos_18 -pos_9 -pos_6 -pos_11 -pos_1 -pos_21 -pos_19 -pos_22 -pos_7 -pos_8 -pos_12 -pos_14 -pos_16 -pos_3 -pos_2 -pos_9 -pos_14 -pos_3 -pos_1 -pos_16 -pos_8 -pos_5 -pos_4 -pos_15 -pos_17 -pos_12 -pos_11 -pos_10 -pos_18 -pos_7 -pos_6 -pos_13 -pos_18 -pos_23 -pos_20 -pos_13 -pos_25 -pos_5 -pos_6 -pos_21 -pos_1 -pos_8 -pos_7 -pos_9 -pos_16 -pos_12 -pos_17 -pos_2 -pos_19 -pos_24 -pos_15 -pos_10 -pos_3 -pos_22 -pos_4 -pos_11 -pos_14 -pos_9 -pos_6 -pos_1 -pos_10 -pos_4 -pos_8 -pos_7 -pos_12 -pos_16 -pos_5 -pos_11 -pos_14 -pos_3 -pos_2 -pos_15 -pos_13 -pos_11 -pos_4 -pos_10 -pos_7 -pos_9 -pos_6 -pos_2 -pos_8 -pos_12 -pos_3 -pos_1 -pos_5 -pos_8 -pos_6 -pos_12 -pos_9 -pos_3 -pos_10 -pos_11 -pos_4 -pos_2 -pos_1 -pos_7 -pos_5 -pos_6 -pos_3 -pos_2 -pos_10 -pos_8 -pos_1 -pos_15 -pos_9 -pos_4 -pos_13 -pos_14 -pos_5 -pos_11 -pos_12 -pos_7 -pos_14 -pos_1 -pos_13 -pos_11 -pos_8 -pos_7 -pos_10 -pos_9 -pos_5 -pos_3 -pos_15 -pos_2 -pos_6 -pos_12 -pos_4 -pos_22 -pos_14 -pos_12 -pos_11 -pos_2 -pos_24 -pos_7 -pos_15 -pos_19 -pos_5 -pos_3 -pos_6 -pos_25 -pos_9 -pos_4 -pos_23 -pos_21 -pos_20 -pos_8 -pos_16 -pos_18 -pos_1 -pos_13 -pos_17 -pos_10 -pos_13 -pos_8 -pos_10 -pos_1 -pos_3 -pos_9 -pos_5 -pos_11 -pos_7 -pos_12 -pos_6 -pos_4 -pos_2 -pos_4 -pos_7 -pos_1 -pos_3 -pos_9 -pos_6 -pos_8 -pos_5 -pos_2 -pos_7 -pos_4 -pos_3 -pos_6 -pos_8 -pos_12 -pos_9 -pos_5 -pos_2 -pos_10 -pos_1 -pos_11 -pos_8 -pos_5 -pos_7 -pos_3 -pos_6 -pos_4 -pos_1 -pos_2 -pos_9 -pos_16 -pos_10 -pos_5 -pos_13 -pos_2 -pos_11 -pos_3 -pos_1 -pos_12 -pos_14 -pos_4 -pos_8 -pos_7 -pos_6 -pos_15 -pos_4 -pos_5 -pos_23 -pos_14 -pos_3 -pos_22 -pos_7 -pos_20 -pos_8 -pos_18 -pos_25 -pos_2 -pos_17 -pos_6 -pos_15 -pos_12 -pos_13 -pos_10 -pos_1 -pos_24 -pos_11 -pos_19 -pos_9 -pos_21 -pos_16 -pos_16 -pos_13 -pos_5 -pos_9 -pos_10 -pos_3 -pos_7 -pos_14 -pos_4 -pos_17 -pos_12 -pos_19 -pos_8 -pos_6 -pos_18 -pos_11 -pos_2 -pos_1 -pos_15 -pos_19 -pos_17 -pos_6 -pos_12 -pos_15 -pos_14 -pos_2 -pos_24 -pos_5 -pos_13 -pos_21 -pos_10 -pos_9 -pos_8 -pos_1 -pos_4 -pos_11 -pos_22 -pos_7 -pos_18 -pos_3 -pos_20 -pos_23 -pos_16 -pos_6 -pos_9 -pos_12 -pos_7 -pos_2 -pos_3 -pos_5 -pos_10 -pos_4 -pos_11 -pos_8 -pos_1 -pos_4 -pos_10 -pos_14 -pos_12 -pos_15 -pos_13 -pos_2 -pos_5 -pos_16 -pos_1 -pos_3 -pos_8 -pos_11 -pos_9 -pos_7 -pos_6 -pos_2 -pos_5 -pos_7 -pos_6 -pos_3 -pos_9 -pos_4 -pos_10 -pos_1 -pos_8 -pos_9 -pos_6 -pos_8 -pos_5 -pos_7 -pos_3 -pos_15 -pos_13 -pos_10 -pos_2 -pos_11 -pos_1 -pos_14 -pos_4 -pos_16 -pos_12 -pos_9 -pos_5 -pos_2 -pos_3 -pos_6 -pos_7 -pos_1 -pos_4 -pos_8 -pos_9 -pos_5 -pos_6 -pos_3 -pos_17 -pos_12 -pos_7 -pos_10 -pos_4 -pos_1 -pos_2 -pos_11 -pos_14 -pos_8 -pos_15 -pos_13 -pos_16 -pos_12 -pos_8 -pos_5 -pos_4 -pos_6 -pos_1 -pos_7 -pos_13 -pos_11 -pos_9 -pos_14 -pos_2 -pos_10 -pos_15 -pos_3 -pos_7 -pos_5 -pos_6 -pos_4 -pos_2 -pos_3 -pos_1 -pos_3 -pos_9 -pos_2 -pos_7 -pos_5 -pos_4 -pos_8 -pos_6 -pos_1 -pos_9 -pos_10 -pos_4 -pos_5 -pos_6 -pos_11 -pos_12 -pos_2 -pos_1 -pos_8 -pos_14 -pos_15 -pos_7 -pos_3 -pos_13 -pos_7 -pos_9 -pos_6 -pos_11 -pos_12 -pos_1 -pos_14 -pos_3 -pos_13 -pos_5 -pos_2 -pos_10 -pos_8 -pos_4 -pos_1 -pos_3 -pos_2 -pos_10 -pos_11 -pos_9 -pos_4 -pos_8 -pos_12 -pos_14 -pos_16 -pos_6 -pos_15 -pos_5 -pos_7 -pos_13 -pos_13 -pos_6 -pos_2 -pos_7 -pos_5 -pos_3 -pos_12 -pos_17 -pos_14 -pos_10 -pos_9 -pos_1 -pos_16 -pos_8 -pos_15 -pos_4 -pos_11 -pos_6 -pos_8 -pos_3 -pos_4 -pos_2 -pos_7 -pos_1 -pos_5 -pos_7 -pos_11 -pos_1 -pos_15 -pos_2 -pos_5 -pos_8 -pos_6 -pos_9 -pos_13 -pos_14 -pos_4 -pos_10 -pos_3 -pos_16 -pos_17 -pos_12 -pos_5 -pos_7 -pos_1 -pos_10 -pos_3 -pos_9 -pos_4 -pos_8 -pos_2 -pos_12 -pos_11 -pos_6 -pos_17 -pos_13 -pos_8 -pos_2 -pos_4 -pos_9 -pos_18 -pos_14 -pos_12 -pos_15 -pos_10 -pos_16 -pos_5 -pos_1 -pos_11 -pos_7 -pos_6 -pos_3 -pos_11 -pos_23 -pos_15 -pos_9 -pos_8 -pos_6 -pos_14 -pos_7 -pos_5 -pos_4 -pos_16 -pos_21 -pos_12 -pos_10 -pos_13 -pos_19 -pos_2 -pos_22 -pos_17 -pos_1 -pos_20 -pos_3 -pos_18 -pos_10 -pos_5 -pos_7 -pos_3 -pos_6 -pos_8 -pos_9 -pos_1 -pos_2 -pos_4 -pos_4 -pos_1 -pos_3 -pos_6 -pos_9 -pos_5 -pos_8 -pos_10 -pos_12 -pos_7 -pos_2 -pos_11 -pos_8 -pos_11 -pos_4 -pos_10 -pos_12 -pos_7 -pos_1 -pos_3 -pos_9 -pos_6 -pos_5 -pos_2 -pos_4 -pos_3 -pos_2 -pos_1 -pos_6 -pos_11 -pos_12 -pos_4 -pos_14 -pos_2 -pos_3 -pos_1 -pos_7 -pos_10 -pos_8 -pos_5 -pos_9 -pos_13 -pos_7 -pos_3 -pos_4 -pos_6 -pos_8 -pos_10 -pos_9 -pos_1 -pos_12 -pos_5 -pos_2 -pos_11 -pos_4 -pos_3 -pos_10 -pos_5 -pos_1 -pos_2 -pos_15 -pos_7 -pos_8 -pos_14 -pos_6 -pos_11 -pos_9 -pos_12 -pos_13 -pos_7 -pos_23 -pos_12 -pos_19 -pos_1 -pos_27 -pos_8 -pos_21 -pos_17 -pos_5 -pos_11 -pos_14 -pos_4 -pos_16 -pos_6 -pos_9 -pos_3 -pos_25 -pos_22 -pos_10 -pos_18 -pos_20 -pos_26 -pos_2 -pos_15 -pos_24 -pos_13 -pos_3 -pos_16 -pos_6 -pos_10 -pos_12 -pos_14 -pos_2 -pos_11 -pos_5 -pos_15 -pos_13 -pos_7 -pos_4 -pos_8 -pos_9 -pos_1 -pos_15 -pos_3 -pos_16 -pos_18 -pos_2 -pos_4 -pos_7 -pos_5 -pos_12 -pos_19 -pos_6 -pos_20 -pos_9 -pos_13 -pos_10 -pos_11 -pos_17 -pos_14 -pos_1 -pos_8 -pos_12 -pos_8 -pos_5 -pos_13 -pos_11 -pos_4 -pos_2 -pos_1 -pos_3 -pos_10 -pos_9 -pos_6 -pos_7 -pos_4 -pos_6 -pos_14 -pos_2 -pos_15 -pos_18 -pos_11 -pos_5 -pos_8 -pos_10 -pos_9 -pos_19 -pos_17 -pos_16 -pos_13 -pos_12 -pos_1 -pos_7 -pos_3 -pos_4 -pos_11 -pos_9 -pos_10 -pos_6 -pos_13 -pos_12 -pos_5 -pos_2 -pos_7 -pos_1 -pos_3 -pos_8 -pos_7 -pos_11 -pos_5 -pos_3 -pos_8 -pos_6 -pos_16 -pos_1 -pos_4 -pos_15 -pos_2 -pos_13 -pos_17 -pos_12 -pos_14 -pos_9 -pos_10 -pos_1 -pos_14 -pos_13 -pos_15 -pos_8 -pos_12 -pos_16 -pos_6 -pos_17 -pos_11 -pos_4 -pos_5 -pos_3 -pos_9 -pos_10 -pos_2 -pos_7 -pos_6 -pos_5 -pos_16 -pos_8 -pos_7 -pos_3 -pos_11 -pos_10 -pos_4 -pos_14 -pos_15 -pos_12 -pos_13 -pos_2 -pos_1 -pos_9 -pos_7 -pos_8 -pos_6 -pos_1 -pos_5 -pos_4 -pos_10 -pos_12 -pos_3 -pos_9 -pos_11 -pos_2 -pos_13 -pos_6 -pos_9 -pos_4 -pos_2 -pos_3 -pos_11 -pos_15 -pos_5 -pos_7 -pos_14 -pos_12 -pos_1 -pos_10 -pos_8 -pos_13 -pos_2 -pos_9 -pos_8 -pos_7 -pos_12 -pos_14 -pos_11 -pos_4 -pos_5 -pos_6 -pos_10 -pos_3 -pos_1 -pos_3 -pos_1 -pos_5 -pos_10 -pos_12 -pos_7 -pos_9 -pos_6 -pos_14 -pos_4 -pos_11 -pos_8 -pos_13 -pos_2 -pos_11 -pos_2 -pos_4 -pos_6 -pos_12 -pos_14 -pos_15 -pos_16 -pos_13 -pos_18 -pos_17 -pos_8 -pos_9 -pos_7 -pos_19 -pos_1 -pos_3 -pos_5 -pos_10 -pos_6 -pos_7 -pos_11 -pos_8 -pos_9 -pos_2 -pos_10 -pos_5 -pos_1 -pos_3 -pos_4 -pos_12 -pos_5 -pos_4 -pos_14 -pos_9 -pos_10 -pos_8 -pos_7 -pos_19 -pos_21 -pos_11 -pos_18 -pos_16 -pos_20 -pos_15 -pos_1 -pos_13 -pos_12 -pos_6 -pos_23 -pos_2 -pos_17 -pos_22 -pos_3 -pos_7 -pos_17 -pos_4 -pos_8 -pos_9 -pos_6 -pos_11 -pos_10 -pos_2 -pos_15 -pos_12 -pos_16 -pos_1 -pos_18 -pos_14 -pos_5 -pos_3 -pos_13 -pos_13 -pos_11 -pos_7 -pos_2 -pos_15 -pos_1 -pos_4 -pos_9 -pos_12 -pos_6 -pos_3 -pos_10 -pos_8 -pos_14 -pos_5 -pos_16 -pos_1 -pos_7 -pos_8 -pos_6 -pos_2 -pos_9 -pos_3 -pos_4 -pos_5 -pos_1 -pos_7 -pos_2 -pos_6 -pos_19 -pos_22 -pos_21 -pos_10 -pos_15 -pos_12 -pos_11 -pos_20 -pos_23 -pos_3 -pos_5 -pos_16 -pos_18 -pos_17 -pos_13 -pos_9 -pos_8 -pos_14 -pos_4 -pos_2 -pos_6 -pos_1 -pos_4 -pos_9 -pos_8 -pos_10 -pos_7 -pos_3 -pos_5 -pos_11 -pos_6 -pos_3 -pos_13 -pos_10 -pos_2 -pos_8 -pos_4 -pos_12 -pos_11 -pos_14 -pos_5 -pos_9 -pos_7 -pos_1 -pos_15 -pos_8 -pos_6 -pos_7 -pos_4 -pos_5 -pos_3 -pos_2 -pos_1 -pos_1 -pos_10 -pos_2 -pos_5 -pos_9 -pos_6 -pos_3 -pos_7 -pos_8 -pos_4 -pos_5 -pos_8 -pos_3 -pos_1 -pos_6 -pos_9 -pos_10 -pos_4 -pos_2 -pos_7 -pos_16 -pos_11 -pos_15 -pos_12 -pos_8 -pos_14 -pos_5 -pos_9 -pos_13 -pos_4 -pos_10 -pos_2 -pos_1 -pos_7 -pos_6 -pos_3 -pos_2 -pos_11 -pos_9 -pos_4 -pos_10 -pos_6 -pos_7 -pos_5 -pos_3 -pos_8 -pos_1 -pos_4 -pos_10 -pos_3 -pos_2 -pos_5 -pos_6 -pos_15 -pos_14 -pos_12 -pos_1 -pos_13 -pos_8 -pos_9 -pos_7 -pos_11 -pos_18 -pos_21 -pos_9 -pos_12 -pos_8 -pos_17 -pos_5 -pos_10 -pos_4 -pos_16 -pos_1 -pos_22 -pos_20 -pos_13 -pos_14 -pos_2 -pos_15 -pos_3 -pos_7 -pos_6 -pos_19 -pos_11 -pos_16 -pos_1 -pos_11 -pos_15 -pos_5 -pos_10 -pos_3 -pos_12 -pos_14 -pos_4 -pos_13 -pos_8 -pos_2 -pos_9 -pos_7 -pos_6 -pos_13 -pos_11 -pos_1 -pos_14 -pos_4 -pos_6 -pos_3 -pos_9 -pos_2 -pos_7 -pos_8 -pos_12 -pos_5 -pos_10 -pos_17 -pos_15 -pos_16 -pos_21 -pos_19 -pos_13 -pos_16 -pos_5 -pos_14 -pos_1 -pos_23 -pos_4 -pos_11 -pos_6 -pos_9 -pos_2 -pos_10 -pos_20 -pos_8 -pos_15 -pos_17 -pos_12 -pos_3 -pos_7 -pos_22 -pos_18 -pos_12 -pos_9 -pos_5 -pos_13 -pos_7 -pos_15 -pos_6 -pos_3 -pos_8 -pos_14 -pos_16 -pos_4 -pos_1 -pos_11 -pos_10 -pos_2 -pos_9 -pos_1 -pos_15 -pos_17 -pos_8 -pos_4 -pos_6 -pos_22 -pos_21 -pos_20 -pos_12 -pos_13 -pos_19 -pos_11 -pos_16 -pos_18 -pos_7 -pos_5 -pos_2 -pos_14 -pos_3 -pos_10 -pos_11 -pos_6 -pos_12 -pos_7 -pos_17 -pos_3 -pos_4 -pos_10 -pos_1 -pos_9 -pos_15 -pos_14 -pos_5 -pos_2 -pos_13 -pos_16 -pos_8 -pos_12 -pos_3 -pos_11 -pos_5 -pos_6 -pos_10 -pos_8 -pos_2 -pos_13 -pos_1 -pos_14 -pos_7 -pos_4 -pos_9 -pos_12 -pos_6 -pos_8 -pos_11 -pos_9 -pos_2 -pos_1 -pos_10 -pos_4 -pos_3 -pos_7 -pos_5 -pos_10 -pos_4 -pos_12 -pos_1 -pos_8 -pos_9 -pos_3 -pos_5 -pos_13 -pos_2 -pos_11 -pos_6 -pos_7 -pos_14 -pos_11 -pos_18 -pos_6 -pos_15 -pos_20 -pos_4 -pos_16 -pos_5 -pos_13 -pos_17 -pos_3 -pos_10 -pos_8 -pos_12 -pos_19 -pos_14 -pos_2 -pos_9 -pos_1 -pos_7 -pos_8 -pos_12 -pos_3 -pos_14 -pos_5 -pos_4 -pos_10 -pos_9 -pos_7 -pos_11 -pos_15 -pos_2 -pos_13 -pos_1 -pos_6 -pos_14 -pos_15 -pos_1 -pos_3 -pos_11 -pos_13 -pos_17 -pos_8 -pos_6 -pos_9 -pos_4 -pos_12 -pos_7 -pos_10 -pos_5 -pos_2 -pos_16 -pos_13 -pos_9 -pos_12 -pos_2 -pos_1 -pos_8 -pos_5 -pos_15 -pos_11 -pos_3 -pos_4 -pos_7 -pos_6 -pos_14 -pos_10 -pos_14 -pos_11 -pos_1 -pos_4 -pos_3 -pos_7 -pos_5 -pos_10 -pos_13 -pos_6 -pos_15 -pos_2 -pos_12 -pos_8 -pos_9 -pos_22 -pos_5 -pos_16 -pos_1 -pos_18 -pos_20 -pos_17 -pos_14 -pos_15 -pos_10 -pos_9 -pos_11 -pos_21 -pos_2 -pos_13 -pos_7 -pos_8 -pos_19 -pos_12 -pos_6 -pos_4 -pos_3 -pos_2 -pos_1 -pos_8 -pos_9 -pos_3 -pos_7 -pos_5 -pos_4 -pos_6 -pos_13 -pos_3 -pos_2 -pos_10 -pos_14 -pos_1 -pos_17 -pos_7 -pos_21 -pos_15 -pos_4 -pos_9 -pos_20 -pos_19 -pos_8 -pos_16 -pos_5 -pos_12 -pos_6 -pos_18 -pos_11 -pos_8 -pos_2 -pos_1 -pos_5 -pos_7 -pos_9 -pos_6 -pos_3 -pos_4 -pos_9 -pos_8 -pos_4 -pos_12 -pos_17 -pos_15 -pos_11 -pos_5 -pos_14 -pos_1 -pos_3 -pos_7 -pos_13 -pos_10 -pos_6 -pos_16 -pos_2 -pos_15 -pos_10 -pos_14 -pos_16 -pos_7 -pos_6 -pos_9 -pos_8 -pos_11 -pos_2 -pos_12 -pos_5 -pos_4 -pos_1 -pos_13 -pos_3 -pos_4 -pos_6 -pos_7 -pos_10 -pos_8 -pos_5 -pos_1 -pos_15 -pos_13 -pos_2 -pos_14 -pos_11 -pos_9 -pos_12 -pos_3 -pos_9 -pos_2 -pos_12 -pos_13 -pos_6 -pos_3 -pos_4 -pos_5 -pos_7 -pos_1 -pos_10 -pos_11 -pos_8 -pos_5 -pos_7 -pos_3 -pos_10 -pos_4 -pos_8 -pos_1 -pos_9 -pos_2 -pos_11 -pos_6 -pos_12 -pos_13 -pos_14 -pos_11 -pos_6 -pos_2 -pos_4 -pos_5 -pos_13 -pos_15 -pos_9 -pos_1 -pos_12 -pos_8 -pos_7 -pos_3 -pos_10 -pos_4 -pos_5 -pos_8 -pos_3 -pos_2 -pos_7 -pos_1 -pos_13 -pos_6 -pos_9 -pos_14 -pos_12 -pos_10 -pos_11 -pos_10 -pos_11 -pos_18 -pos_6 -pos_8 -pos_12 -pos_13 -pos_5 -pos_3 -pos_16 -pos_17 -pos_2 -pos_7 -pos_9 -pos_4 -pos_14 -pos_15 -pos_1 -pos_15 -pos_7 -pos_3 -pos_10 -pos_21 -pos_20 -pos_2 -pos_8 -pos_16 -pos_6 -pos_5 -pos_13 -pos_18 -pos_11 -pos_4 -pos_12 -pos_17 -pos_14 -pos_1 -pos_9 -pos_19 -pos_6 -pos_9 -pos_5 -pos_3 -pos_11 -pos_2 -pos_10 -pos_8 -pos_7 -pos_14 -pos_12 -pos_13 -pos_1 -pos_4 -pos_7 -pos_15 -pos_14 -pos_10 -pos_4 -pos_2 -pos_11 -pos_8 -pos_16 -pos_5 -pos_12 -pos_6 -pos_9 -pos_1 -pos_3 -pos_17 -pos_13 -pos_2 -pos_1 -pos_10 -pos_7 -pos_14 -pos_12 -pos_8 -pos_3 -pos_15 -pos_6 -pos_4 -pos_13 -pos_11 -pos_5 -pos_9 -pos_6 -pos_1 -pos_14 -pos_10 -pos_2 -pos_5 -pos_7 -pos_9 -pos_13 -pos_4 -pos_11 -pos_8 -pos_3 -pos_12 -pos_13 -pos_8 -pos_3 -pos_2 -pos_11 -pos_5 -pos_7 -pos_10 -pos_4 -pos_6 -pos_12 -pos_15 -pos_14 -pos_9 -pos_16 -pos_1 -pos_5 -pos_10 -pos_12 -pos_2 -pos_1 -pos_7 -pos_8 -pos_4 -pos_3 -pos_9 -pos_11 -pos_6 -pos_11 -pos_6 -pos_3 -pos_8 -pos_14 -pos_16 -pos_17 -pos_13 -pos_15 -pos_4 -pos_12 -pos_1 -pos_10 -pos_2 -pos_9 -pos_5 -pos_7 -pos_3 -pos_14 -pos_7 -pos_16 -pos_4 -pos_17 -pos_1 -pos_15 -pos_5 -pos_6 -pos_12 -pos_19 -pos_8 -pos_13 -pos_11 -pos_9 -pos_18 -pos_10 -pos_2 -pos_15 -pos_8 -pos_16 -pos_12 -pos_6 -pos_7 -pos_5 -pos_10 -pos_3 -pos_1 -pos_4 -pos_2 -pos_13 -pos_14 -pos_9 -pos_11 -pos_4 -pos_5 -pos_10 -pos_8 -pos_2 -pos_6 -pos_11 -pos_7 -pos_9 -pos_1 -pos_3 -pos_10 -pos_13 -pos_8 -pos_18 -pos_4 -pos_2 -pos_7 -pos_9 -pos_17 -pos_6 -pos_16 -pos_12 -pos_14 -pos_1 -pos_3 -pos_11 -pos_5 -pos_15 -pos_5 -pos_8 -pos_9 -pos_1 -pos_2 -pos_11 -pos_7 -pos_10 -pos_3 -pos_4 -pos_6 -pos_5 -pos_13 -pos_2 -pos_9 -pos_3 -pos_1 -pos_8 -pos_10 -pos_4 -pos_12 -pos_11 -pos_7 -pos_6 -pos_8 -pos_3 -pos_5 -pos_13 -pos_4 -pos_11 -pos_1 -pos_9 -pos_7 -pos_10 -pos_14 -pos_6 -pos_12 -pos_2 -pos_4 -pos_9 -pos_3 -pos_1 -pos_5 -pos_2 -pos_6 -pos_8 -pos_7 -pos_11 -pos_2 -pos_13 -pos_4 -pos_7 -pos_15 -pos_14 -pos_1 -pos_9 -pos_6 -pos_3 -pos_8 -pos_10 -pos_5 -pos_12 -pos_16 -pos_1 -pos_11 -pos_10 -pos_9 -pos_12 -pos_5 -pos_6 -pos_7 -pos_3 -pos_14 -pos_13 -pos_4 -pos_15 -pos_2 -pos_8 -pos_12 -pos_11 -pos_15 -pos_2 -pos_4 -pos_1 -pos_10 -pos_5 -pos_6 -pos_9 -pos_8 -pos_13 -pos_3 -pos_16 -pos_7 -pos_14 -pos_11 -pos_7 -pos_17 -pos_9 -pos_1 -pos_5 -pos_3 -pos_2 -pos_19 -pos_18 -pos_24 -pos_6 -pos_8 -pos_16 -pos_12 -pos_20 -pos_15 -pos_22 -pos_10 -pos_25 -pos_21 -pos_4 -pos_23 -pos_14 -pos_13 -pos_4 -pos_7 -pos_3 -pos_1 -pos_9 -pos_8 -pos_5 -pos_2 -pos_6 -pos_11 -pos_2 -pos_12 -pos_13 -pos_10 -pos_9 -pos_1 -pos_6 -pos_4 -pos_8 -pos_7 -pos_5 -pos_3 -pos_15 -pos_7 -pos_11 -pos_13 -pos_19 -pos_2 -pos_6 -pos_8 -pos_12 -pos_16 -pos_22 -pos_17 -pos_18 -pos_4 -pos_1 -pos_14 -pos_20 -pos_3 -pos_21 -pos_9 -pos_10 -pos_5 -pos_13 -pos_8 -pos_9 -pos_6 -pos_2 -pos_10 -pos_7 -pos_12 -pos_15 -pos_16 -pos_11 -pos_1 -pos_5 -pos_14 -pos_4 -pos_3 -pos_5 -pos_3 -pos_14 -pos_18 -pos_10 -pos_13 -pos_8 -pos_15 -pos_1 -pos_9 -pos_11 -pos_7 -pos_12 -pos_16 -pos_4 -pos_6 -pos_2 -pos_17 -pos_2 -pos_16 -pos_6 -pos_20 -pos_13 -pos_12 -pos_14 -pos_17 -pos_5 -pos_10 -pos_19 -pos_18 -pos_3 -pos_8 -pos_1 -pos_4 -pos_7 -pos_15 -pos_9 -pos_11 -pos_8 -pos_14 -pos_1 -pos_6 -pos_10 -pos_5 -pos_3 -pos_13 -pos_12 -pos_4 -pos_2 -pos_11 -pos_9 -pos_7 -pos_8 -pos_11 -pos_4 -pos_7 -pos_3 -pos_1 -pos_5 -pos_9 -pos_6 -pos_10 -pos_2 -pos_3 -pos_6 -pos_1 -pos_4 -pos_2 -pos_5 -pos_9 -pos_7 -pos_8 -pos_11 -pos_2 -pos_15 -pos_10 -pos_8 -pos_9 -pos_5 -pos_16 -pos_4 -pos_13 -pos_3 -pos_7 -pos_1 -pos_6 -pos_12 -pos_14 -pos_15 -pos_7 -pos_9 -pos_6 -pos_14 -pos_19 -pos_8 -pos_11 -pos_18 -pos_5 -pos_10 -pos_13 -pos_17 -pos_1 -pos_12 -pos_2 -pos_3 -pos_4 -pos_16 -pos_13 -pos_2 -pos_17 -pos_8 -pos_16 -pos_11 -pos_1 -pos_15 -pos_4 -pos_10 -pos_9 -pos_14 -pos_18 -pos_3 -pos_5 -pos_6 -pos_7 -pos_12 -pos_19 -pos_5 -pos_7 -pos_3 -pos_1 -pos_10 -pos_6 -pos_11 -pos_2 -pos_9 -pos_8 -pos_4 -pos_9 -pos_6 -pos_8 -pos_4 -pos_5 -pos_3 -pos_11 -pos_7 -pos_2 -pos_10 -pos_1 -pos_3 -pos_1 -pos_12 -pos_9 -pos_2 -pos_13 -pos_7 -pos_10 -pos_4 -pos_6 -pos_8 -pos_11 -pos_5 -pos_12 -pos_7 -pos_10 -pos_4 -pos_2 -pos_1 -pos_11 -pos_9 -pos_14 -pos_13 -pos_6 -pos_3 -pos_5 -pos_8 -pos_8 -pos_6 -pos_14 -pos_5 -pos_4 -pos_7 -pos_9 -pos_12 -pos_13 -pos_10 -pos_3 -pos_1 -pos_2 -pos_11 -pos_1 -pos_2 -pos_6 -pos_4 -pos_10 -pos_11 -pos_5 -pos_13 -pos_3 -pos_12 -pos_7 -pos_9 -pos_8 -pos_3 -pos_1 -pos_10 -pos_5 -pos_8 -pos_2 -pos_4 -pos_6 -pos_15 -pos_11 -pos_13 -pos_9 -pos_7 -pos_16 -pos_12 -pos_14 -pos_4 -pos_5 -pos_2 -pos_6 -pos_3 -pos_1 -pos_4 -pos_16 -pos_20 -pos_11 -pos_2 -pos_12 -pos_3 -pos_19 -pos_21 -pos_8 -pos_10 -pos_15 -pos_13 -pos_18 -pos_6 -pos_17 -pos_7 -pos_1 -pos_5 -pos_14 -pos_9 -pos_14 -pos_11 -pos_7 -pos_8 -pos_10 -pos_12 -pos_2 -pos_6 -pos_5 -pos_4 -pos_9 -pos_13 -pos_3 -pos_16 -pos_1 -pos_15 -pos_1 -pos_5 -pos_11 -pos_9 -pos_2 -pos_3 -pos_6 -pos_4 -pos_7 -pos_10 -pos_8 -pos_12 -pos_3 -pos_8 -pos_4 -pos_5 -pos_16 -pos_1 -pos_15 -pos_11 -pos_6 -pos_10 -pos_2 -pos_14 -pos_9 -pos_13 -pos_12 -pos_7 -pos_9 -pos_3 -pos_11 -pos_4 -pos_8 -pos_1 -pos_6 -pos_7 -pos_2 -pos_10 -pos_5 -pos_21 -pos_7 -pos_23 -pos_2 -pos_24 -pos_15 -pos_19 -pos_13 -pos_8 -pos_16 -pos_12 -pos_4 -pos_11 -pos_9 -pos_20 -pos_22 -pos_18 -pos_5 -pos_10 -pos_6 -pos_1 -pos_17 -pos_3 -pos_14 -pos_8 -pos_6 -pos_1 -pos_11 -pos_5 -pos_7 -pos_4 -pos_3 -pos_9 -pos_10 -pos_2 -pos_12 -pos_7 -pos_3 -pos_5 -pos_4 -pos_10 -pos_6 -pos_8 -pos_2 -pos_9 -pos_1 diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 424081aee344..02c146ca1c01 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -219,9 +219,9 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector 0? score_adjusted.data(): score + start, + GetGradientsForOneQuery(i, cnt, label_ + start, num_position_ids_ > 0 ? score_adjusted.data() : score + start, gradients + start, hessians + start); if (weights_ != nullptr) { for (data_size_t j = 0; j < cnt; ++j) { diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index fc7b702405c9..ae83cd43dd58 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -764,7 +764,12 @@ def test_ranking_with_position_information(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - copyfile(str(rank_example_dir / '_rank.train.position'), str(tmp_path / 'rank.train.position')) + with open(str(tmp_path / 'rank.train.position'), "w") as out_file: + np.random.seed(0) + for _ in range(lgb_train.num_data()): + position = np.random.randint(28) + out_file.write(f"pos_{position}\n") + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) @@ -778,7 +783,7 @@ def test_ranking_with_position_information(tmp_path): file.close() lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] - with pytest.raises(lgb.basic.LightGBMError, match="Positions size doesn't match data size"): + with pytest.raises(lgb.basic.LightGBMError, match="Positions size \(3006\) doesn't match data size"): lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) def test_early_stopping(): From 923b41a0a8e8b7fd415805aa382b6ff9badbd8ca Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Wed, 12 Jul 2023 12:29:25 +0000 Subject: [PATCH 43/97] add position in python API --- include/LightGBM/dataset.h | 8 +-- python-package/lightgbm/basic.py | 67 ++++++++++++++++++++++-- src/io/dataset.cpp | 5 ++ src/io/metadata.cpp | 46 ++++++++++++++-- src/objective/rank_objective.hpp | 4 +- tests/python_package_test/test_engine.py | 14 +++-- 6 files changed, 126 insertions(+), 18 deletions(-) diff --git a/include/LightGBM/dataset.h b/include/LightGBM/dataset.h index 6492a75fa4f5..e7baa42dc2e6 100644 --- a/include/LightGBM/dataset.h +++ b/include/LightGBM/dataset.h @@ -114,6 +114,8 @@ class Metadata { void SetQuery(const data_size_t* query, data_size_t len); + void SetPosition(const data_size_t* position, data_size_t len); + /*! * \brief Set initial scores * \param init_score Initial scores, this class will manage memory for init_score. @@ -217,7 +219,7 @@ class Metadata { * \brief Get positions, if does not exist then return nullptr * \return Pointer of positions */ - inline const size_t* positions() const { + inline const data_size_t* positions() const { if (!positions_.empty()) { return positions_.data(); } else { @@ -241,7 +243,7 @@ class Metadata { * \brief Get Number of different position IDs * \return number of different position IDs */ - inline const size_t num_position_ids() const { + inline size_t num_position_ids() const { return position_ids_.size(); } @@ -350,7 +352,7 @@ class Metadata { /*! \brief Weights data */ std::vector weights_; /*! \brief Positions data */ - std::vector positions_; + std::vector positions_; /*! \brief Position identifiers */ std::vector position_ids_; /*! \brief Query boundaries */ diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index fd07283aa236..23569c608ccd 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -61,6 +61,12 @@ np.ndarray, pd_Series ] +_LGBM_PositionType = Union[ + List[float], + List[int], + np.ndarray, + pd_Series +] _LGBM_InitScoreType = Union[ List[float], List[List[float]], @@ -577,7 +583,8 @@ def _choose_param_value(main_param_name: str, params: Dict[str, Any], default_va "label": _C_API_DTYPE_FLOAT32, "weight": _C_API_DTYPE_FLOAT32, "init_score": _C_API_DTYPE_FLOAT64, - "group": _C_API_DTYPE_INT32 + "group": _C_API_DTYPE_INT32, + "position": _C_API_DTYPE_INT32 } """String name to int feature importance type mapper""" @@ -1473,6 +1480,7 @@ def __init__( reference: Optional["Dataset"] = None, weight: Optional[_LGBM_WeightType] = None, group: Optional[_LGBM_GroupType] = None, + position: Optional[_LGBM_PositionType] = None, init_score: Optional[_LGBM_InitScoreType] = None, feature_name: _LGBM_FeatureNameConfiguration = 'auto', categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto', @@ -1498,6 +1506,8 @@ def __init__( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. + position : list, numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. feature_name : list of str, or 'auto', optional (default="auto") @@ -1524,6 +1534,7 @@ def __init__( self.reference = reference self.weight = weight self.group = group + self.position = position self.init_score = init_score self.feature_name: _LGBM_FeatureNameConfiguration = feature_name self.categorical_feature: _LGBM_CategoricalFeatureConfiguration = categorical_feature @@ -1784,6 +1795,7 @@ def _lazy_init( reference: Optional["Dataset"], weight: Optional[_LGBM_WeightType], group: Optional[_LGBM_GroupType], + position: Optional[_LGBM_PositionType], init_score: Optional[_LGBM_InitScoreType], predictor: Optional[_InnerPredictor], feature_name: _LGBM_FeatureNameConfiguration, @@ -1877,6 +1889,8 @@ def _lazy_init( self.set_weight(weight) if group is not None: self.set_group(group) + if position is not None: + self.set_position(position) if isinstance(predictor, _InnerPredictor): if self._predictor is None and init_score is not None: _log_warning("The init_score will be overridden by the prediction of init_model.") @@ -2171,7 +2185,7 @@ def construct(self) -> "Dataset": if self.used_indices is None: # create valid self._lazy_init(data=self.data, label=self.label, reference=self.reference, - weight=self.weight, group=self.group, + weight=self.weight, group=self.group, position=self.position, init_score=self.init_score, predictor=self._predictor, feature_name=self.feature_name, categorical_feature='auto', params=self.params) else: @@ -2182,6 +2196,10 @@ def construct(self) -> "Dataset": group_info = np.array(self.reference.group).astype(np.int32, copy=False) _, self.group = np.unique(np.repeat(range(len(group_info)), repeats=group_info)[self.used_indices], return_counts=True) + if self.reference.position is not None: + position_info = np.array(self.reference.position).astype(np.int32, copy=False) + _, self.position = np.unique(np.repeat(range(len(position_info)), repeats=position_info)[self.used_indices], + return_counts=True) self.handle = ctypes.c_void_p() params_str = _param_dict_to_str(self.params) _safe_call(_LIB.LGBM_DatasetGetSubset( @@ -2194,6 +2212,8 @@ def construct(self) -> "Dataset": self.get_data() if self.group is not None: self.set_group(self.group) + if self.position is not None: + self.set_position(self.position) if self.get_label() is None: raise ValueError("Label should not be None.") if isinstance(self._predictor, _InnerPredictor) and self._predictor is not self.reference._predictor: @@ -2206,7 +2226,7 @@ def construct(self) -> "Dataset": else: # create train self._lazy_init(data=self.data, label=self.label, reference=None, - weight=self.weight, group=self.group, + weight=self.weight, group=self.group, position=self.position, init_score=self.init_score, predictor=self._predictor, feature_name=self.feature_name, categorical_feature=self.categorical_feature, params=self.params) if self.free_raw_data: @@ -2220,6 +2240,7 @@ def create_valid( label: Optional[_LGBM_LabelType] = None, weight: Optional[_LGBM_WeightType] = None, group: Optional[_LGBM_GroupType] = None, + position: Optional[_LGBM_PositionType] = None, init_score: Optional[_LGBM_InitScoreType] = None, params: Optional[Dict[str, Any]] = None ) -> "Dataset": @@ -2240,6 +2261,8 @@ def create_valid( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. + position : list, numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. params : dict or None, optional (default=None) @@ -2251,7 +2274,7 @@ def create_valid( Validation Dataset with reference to self. """ ret = Dataset(data, label=label, reference=self, - weight=weight, group=group, init_score=init_score, + weight=weight, group=group, position=position, init_score=init_score, params=params, free_raw_data=self.free_raw_data) ret._predictor = self._predictor ret.pandas_categorical = self.pandas_categorical @@ -2386,7 +2409,7 @@ def set_field( 'In multiclass classification init_score can also be a list of lists, numpy 2-D array or pandas DataFrame.' ) else: - dtype = np.int32 if field_name == 'group' else np.float32 + dtype = np.int32 if (field_name == 'group' or field_name == 'position') else np.float32 data = _list_to_1d_numpy(data, dtype=dtype, name=field_name) ptr_data: Union[_ctypes_float_ptr, _ctypes_int_ptr] @@ -2679,6 +2702,28 @@ def set_group( self.set_field('group', group) return self + def set_position( + self, + position: Optional[_LGBM_PositionType] + ) -> "Dataset": + """Set position of Dataset (used for ranking). + + Parameters + ---------- + position : list, numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. + + Returns + ------- + self : Dataset + Dataset with set position. + """ + self.position = position + if self.handle is not None and position is not None: + position = _list_to_1d_numpy(position, dtype=np.int32, name='position') + self.set_field('position', position) + return self + def get_feature_name(self) -> List[str]: """Get the names of columns (features) in the Dataset. @@ -2805,6 +2850,18 @@ def get_group(self) -> Optional[np.ndarray]: self.group = np.diff(self.group) return self.group + def get_position(self) -> Optional[np.ndarray]: + """Get the position of the Dataset. + + Returns + ------- + position : list, numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. + """ + if self.position is None: + self.position = self.get_field('position') + return self.position + def num_data(self) -> int: """Get the number of rows in the Dataset. diff --git a/src/io/dataset.cpp b/src/io/dataset.cpp index 5b23f01ec3a0..2338053af221 100644 --- a/src/io/dataset.cpp +++ b/src/io/dataset.cpp @@ -937,6 +937,8 @@ bool Dataset::SetIntField(const char* field_name, const int* field_data, name = Common::Trim(name); if (name == std::string("query") || name == std::string("group")) { metadata_.SetQuery(field_data, num_element); + } else if (name == std::string("position")) { + metadata_.SetPosition(field_data, num_element); } else { return false; } @@ -987,6 +989,9 @@ bool Dataset::GetIntField(const char* field_name, data_size_t* out_len, if (name == std::string("query") || name == std::string("group")) { *out_ptr = metadata_.query_boundaries(); *out_len = metadata_.num_queries() + 1; + } else if (name == std::string("position")) { + *out_ptr = metadata_.positions(); + *out_len = num_data_; } else { return false; } diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 02c146ca1c01..9e82c9cc4702 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -266,13 +267,13 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector 0 && num_positions_ != num_all_data) { positions_.clear(); num_positions_ = 0; - Log::Fatal("Positions size (%i) doesn't match data size (%i)", num_positions_, num_data_); + Log::Fatal("Positions size (%i) doesn't match data size (%i)", num_positions_, num_data_); } // get local positions if (!positions_.empty()) { auto old_positions = positions_; num_positions_ = num_data_; - positions_ = std::vector(num_data_); + positions_ = std::vector(num_data_); #pragma omp parallel for schedule(static, 512) for (int i = 0; i < static_cast(used_data_indices.size()); ++i) { positions_[i] = old_positions[used_data_indices[i]]; @@ -518,6 +519,41 @@ void Metadata::SetQuery(const data_size_t* query, data_size_t len) { #endif // USE_CUDA } +void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { + std::lock_guard lock(mutex_); + // save to nullptr + if (positions == nullptr || len == 0) { + positions_.clear(); + num_positions_ = 0; + return; + } + if (num_data_ != len) { + Log::Fatal("Length of positions is not same with #data"); + } + if (positions_.empty()) { positions_.resize(num_data_); } + num_positions_ = num_data_; + + position_load_from_file_ = false; + #ifdef USE_CUDA + Log::Fatal("Positions in learning to rank is not supported in CUDA version yet."); + #endif // USE_CUDA + + #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) + for (data_size_t i = 0; i < num_positions_; ++i) { + positions_[i] = positions[i]; + } + + position_ids_.clear(); + std::set position_set; + for (int position : positions_) { + position_set.insert(position); + } + Log::Warning("position_set.size() = %ld", position_set.size()); + for (int position : position_set) { + position_ids_.push_back(std::to_string(position)); + } +} + void Metadata::InsertQueries(const data_size_t* queries, data_size_t start_index, data_size_t len) { if (!queries) { Log::Fatal("Passed null queries"); @@ -569,13 +605,13 @@ void Metadata::LoadPositions() { } Log::Info("Loading positions..."); num_positions_ = static_cast(reader.Lines().size()); - positions_ = std::vector(num_positions_); + positions_ = std::vector(num_positions_); position_ids_ = std::vector(); - std::unordered_map map_id2pos; + std::unordered_map map_id2pos; for (data_size_t i = 0; i < num_positions_; ++i) { std::string& line = reader.Lines()[i]; if (map_id2pos.count(line) == 0) { - map_id2pos[line] = position_ids_.size(); + map_id2pos[line] = static_cast(position_ids_.size()); position_ids_.push_back(line); } positions_[i] = map_id2pos.at(line); diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 360721c762cf..2ce9c736c74b 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -89,7 +89,7 @@ class RankingObjective : public ObjectiveFunction { const double* score, score_t* lambdas, score_t* hessians) const = 0; - virtual void UpdatePositionBiasFactors(const score_t* lambdas, const score_t* hessians) const {} + virtual void UpdatePositionBiasFactors(const score_t* /*lambdas*/, const score_t* /*hessians*/) const {} const char* GetName() const override = 0; @@ -111,7 +111,7 @@ class RankingObjective : public ObjectiveFunction { /*! \brief Pointer of weights */ const label_t* weights_; /*! \brief Pointer of positions */ - const size_t* positions_; + const data_size_t* positions_; /*! \brief Pointer of position IDs */ const std::string* position_ids_; /*! \brief Pointer of label */ diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index ae83cd43dd58..818c863df0da 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -753,7 +753,7 @@ def test_ranking_with_position_information(tmp_path): 'objective': 'lambdarank', 'verbose': -1, 'eval_at': [3], - 'eval_metric': 'ndcg' + 'metric': 'ndcg' } copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) @@ -764,19 +764,27 @@ def test_ranking_with_position_information(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - with open(str(tmp_path / 'rank.train.position'), "w") as out_file: + positions = [] + with open(str(tmp_path / '_rank.train.position'), "w") as out_file: np.random.seed(0) for _ in range(lgb_train.num_data()): position = np.random.randint(28) + positions.append(position) out_file.write(f"pos_{position}\n") - lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params, position=positions) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + copyfile(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] + # add extra row to position file with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') From 2b91c243d9ade2e91a6615a2063dc3d316fc8809 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Wed, 12 Jul 2023 12:48:23 +0000 Subject: [PATCH 44/97] fix set_positions in basic.py --- python-package/lightgbm/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 67f395299a23..50fe36ee3fe9 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -2722,7 +2722,7 @@ def set_position( Dataset with set position. """ self.position = position - if self.handle is not None and position is not None: + if self._handle is not None and position is not None: position = _list_to_1d_numpy(position, dtype=np.int32, name='position') self.set_field('position', position) return self From 465f9faea516b486919f7124f59818af19ecf0da Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:53:13 -0700 Subject: [PATCH 45/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index d1787b998479..49c3971ed845 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -77,3 +77,26 @@ Recommendations for gcc Users (MinGW, \*nix) -------------------------------------------- - Refer to `gcc Tips <./gcc-Tips.rst>`__. + +Support for Position Bias Treatment +-------------------------------------------- + +Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position when having been presented to a user. LightGBM uses an additional file to store position data, like the following: + +:: + + 3 + 4 + 1 + ... + +It means the position of the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode a position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). + +The position file corresponds with training data file line by line, and has one position per line. + +And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. +In this case, LightGBM will load the position file automatically if it exists. + +Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. + +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which yields to jointly learning the relevance component ``f(x)`` (which is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. From 4cdf7309138cea02bc3ce7a859d642dc6c2e1750 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:55:16 -0700 Subject: [PATCH 46/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 49c3971ed845..c5a0089711c8 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -81,7 +81,7 @@ Recommendations for gcc Users (MinGW, \*nix) Support for Position Bias Treatment -------------------------------------------- -Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position when having been presented to a user. LightGBM uses an additional file to store position data, like the following: +Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position on the screen when having been presented to a user. LightGBM uses an additional file to store position data, like the following: :: From 343968fcff1b486129dab71b4bf9703ae21db9f4 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:00:21 -0700 Subject: [PATCH 47/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index c5a0089711c8..881b6d6e0a5a 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -81,7 +81,7 @@ Recommendations for gcc Users (MinGW, \*nix) Support for Position Bias Treatment -------------------------------------------- -Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position on the screen when having been presented to a user. LightGBM uses an additional file to store position data, like the following: +Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position on the screen when having been presented to a user. LightGBM uses an additional file to store positional data, like the following: :: @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode a position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode a position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). The position file corresponds with training data file line by line, and has one position per line. From 01869067a9d46f91f58d21725593c5dab5b95f10 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:01:06 -0700 Subject: [PATCH 48/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 881b6d6e0a5a..1c5a7d8b00f2 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode a position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode the position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). The position file corresponds with training data file line by line, and has one position per line. From 0819f4a7da5dbfcf54a199174b3ac1d070a91359 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:03:09 -0700 Subject: [PATCH 49/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 1c5a7d8b00f2..9b3c96c0ca91 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which yields to jointly learning the relevance component ``f(x)`` (which is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which yields to jointly learning the relevance component ``f(x)`` (which is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. From 4e54f7090746656c3066773b04c9a6e214216305 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:04:40 -0700 Subject: [PATCH 50/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 9b3c96c0ca91..23c3999bbc07 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which yields to jointly learning the relevance component ``f(x)`` (which is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. From 36f44f1c6fae2e773046575188535bde407d3d61 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:10:36 -0700 Subject: [PATCH 51/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 23c3999bbc07..dc0069768614 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. The positions could be any (arbitrary) integer number and could encode the position of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. A position could be any (arbitrary) integer number and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). The position file corresponds with training data file line by line, and has one position per line. From acfde2cf95f208327fbf27ac4e09b2e15e237ec1 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:12:32 -0700 Subject: [PATCH 52/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index dc0069768614..5fe16f677320 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into a sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. From a7d0ae5f97b51c38103fc4672909f07f22d32516 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 00:30:02 -0700 Subject: [PATCH 53/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 5fe16f677320..d2beaacb6838 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise intercations) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) for subsequent analysis and interpretation of their effects in the trained models. From e8e1854056867c662ff90c8259203f0466d3fb06 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 01:10:37 -0700 Subject: [PATCH 54/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index d2beaacb6838..330f08ba566e 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea for the general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) for subsequent analysis and interpretation of their effects in the trained models. From 108f630976354ce6769528034153aab781751dff Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 01:12:28 -0700 Subject: [PATCH 55/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 330f08ba566e..8d437bd76e55 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -99,4 +99,4 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) to the overall score, for subsequent analysis and interpretation of their effects in the trained models. From 482d394fd18b4df78cf022c2cdb4632aebdbe06c Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Thu, 13 Jul 2023 23:55:37 +0800 Subject: [PATCH 56/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 8d437bd76e55..310bedaed0a0 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -79,7 +79,7 @@ Recommendations for gcc Users (MinGW, \*nix) - Refer to `gcc Tips <./gcc-Tips.rst>`__. Support for Position Bias Treatment --------------------------------------------- +------------------------------------ Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position on the screen when having been presented to a user. LightGBM uses an additional file to store positional data, like the following: From 03e41bd95c89a398a156880edceb50fdb64cf2d8 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:16:42 -0700 Subject: [PATCH 57/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 310bedaed0a0..8b986e17a74e 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -97,6 +97,6 @@ The position file corresponds with training data file line by line, and has one And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. In this case, LightGBM will load the position file automatically if it exists. -Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__ in above. +Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__. Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) to the overall score, for subsequent analysis and interpretation of their effects in the trained models. From 621c1875367e0bd33f48a0d8edb66bbd901663a4 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:20:06 -0700 Subject: [PATCH 58/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 8b986e17a74e..dbd7c89b1a52 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. A position could be any (arbitrary) integer number and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). The position file corresponds with training data file line by line, and has one position per line. @@ -99,4 +99,8 @@ In this case, LightGBM will load the position file automatically if it exists. Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__. -Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) to the overall score, for subsequent analysis and interpretation of their effects in the trained models. +Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. +During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. +Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). +In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels. +Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions `_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) to the overall score, for subsequent analysis and interpretation of their effects in the trained models. From d8adb4cf1f5614fbd6ba676dcb15951f474dd485 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:21:18 -0700 Subject: [PATCH 59/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index dbd7c89b1a52..58a34b3bc571 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was ``3``, second was ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). The position file corresponds with training data file line by line, and has one position per line. From b93ecca460ab233db3a7ff6783c268d9de112cba Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:56:49 -0700 Subject: [PATCH 60/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 58a34b3bc571..7bcc1bc9cd68 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -90,7 +90,7 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts (it is up to the LightGBM user to perform such encoding/enumeration). +It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consiting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). The position file corresponds with training data file line by line, and has one position per line. From 68ffab8cf5991cd12f7219d983fc4ef6c6760305 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Thu, 13 Jul 2023 21:01:09 -0700 Subject: [PATCH 61/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 7bcc1bc9cd68..db42216fa561 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -81,7 +81,8 @@ Recommendations for gcc Users (MinGW, \*nix) Support for Position Bias Treatment ------------------------------------ -Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position on the screen when having been presented to a user. LightGBM uses an additional file to store positional data, like the following: +Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user. +LightGBM uses an additional file to store positional data, like the following: :: @@ -90,7 +91,8 @@ Often the relevance labels provided in Learning-to-Rank taks might be derived fr 1 ... -It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consiting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). +It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. +A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consiting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). The position file corresponds with training data file line by line, and has one position per line. From 81b5f09c69f041f206bbf69cb897108a9cc72db4 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Thu, 20 Jul 2023 14:54:34 +0000 Subject: [PATCH 62/97] remove List from _LGBM_PositionType --- python-package/lightgbm/basic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 50fe36ee3fe9..91db4cfb1a08 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -62,8 +62,6 @@ pd_Series ] _LGBM_PositionType = Union[ - List[float], - List[int], np.ndarray, pd_Series ] From 0db4caf0a29ad43231b160f1f9bf8e6c627f8c1c Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Thu, 20 Jul 2023 14:57:55 +0000 Subject: [PATCH 63/97] move new position parameter to the last in Dataset constructor --- python-package/lightgbm/basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 91db4cfb1a08..120ea353bb95 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -1481,12 +1481,12 @@ def __init__( reference: Optional["Dataset"] = None, weight: Optional[_LGBM_WeightType] = None, group: Optional[_LGBM_GroupType] = None, - position: Optional[_LGBM_PositionType] = None, init_score: Optional[_LGBM_InitScoreType] = None, feature_name: _LGBM_FeatureNameConfiguration = 'auto', categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto', params: Optional[Dict[str, Any]] = None, - free_raw_data: bool = True + free_raw_data: bool = True, + position: Optional[_LGBM_PositionType] = None, ): """Initialize Dataset. From 102fb977f5f30ab7ccad681f6184cb07905b0689 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Fri, 28 Jul 2023 13:20:52 +0000 Subject: [PATCH 64/97] add position_filename as a parameter --- docs/Parameters.rst | 4 ++++ include/LightGBM/config.h | 4 ++++ include/LightGBM/dataset.h | 5 ++++- src/io/config_auto.cpp | 6 ++++++ src/io/dataset_loader.cpp | 4 ++-- src/io/metadata.cpp | 13 ++++++++----- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/Parameters.rst b/docs/Parameters.rst index 0e85091d51cb..db2ae180ed4b 100644 --- a/docs/Parameters.rst +++ b/docs/Parameters.rst @@ -1141,6 +1141,10 @@ Objective Parameters - used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. +- ``position_filename`` :raw-html:`🔗︎`, default = ``""``, type = str + + - path to the file storing positions used in lambdarank + Metric Parameters ----------------- diff --git a/include/LightGBM/config.h b/include/LightGBM/config.h index b96843b3fcc7..a70dd270b12c 100644 --- a/include/LightGBM/config.h +++ b/include/LightGBM/config.h @@ -969,6 +969,10 @@ struct Config { // desc = used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. double lambdarank_position_bias_regularizer = 0.0; + // type = str + // desc = path to the file storing positions used in lambdarank + std::string position_filename = ""; + #ifndef __NVCC__ #pragma endregion diff --git a/include/LightGBM/dataset.h b/include/LightGBM/dataset.h index e7baa42dc2e6..e1c49f2539da 100644 --- a/include/LightGBM/dataset.h +++ b/include/LightGBM/dataset.h @@ -52,8 +52,9 @@ class Metadata { /*! * \brief Initialization will load query level information, since it is need for sampling data * \param data_filename Filename of data + * \param position_filename Filename of positions */ - void Init(const char* data_filename); + void Init(const char* data_filename, const char* position_filename); /*! * \brief init as subset * \param metadata Filename of data @@ -341,6 +342,8 @@ class Metadata { void InsertQueries(const data_size_t* queries, data_size_t start_index, data_size_t len); /*! \brief Filename of current data */ std::string data_filename_; + /*! \brief Filename of ranking positions */ + std::string position_filename_; /*! \brief Number of data */ data_size_t num_data_; /*! \brief Number of weights, used to check correct weight file */ diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp index 746ddafa46b9..0925596fe5a3 100644 --- a/src/io/config_auto.cpp +++ b/src/io/config_auto.cpp @@ -305,6 +305,7 @@ const std::unordered_set& Config::parameter_set() { "lambdarank_norm", "label_gain", "lambdarank_position_bias_regularizer", + "position_filename", "metric", "metric_freq", "is_provide_training_metric", @@ -623,6 +624,8 @@ void Config::GetMembersFromString(const std::unordered_map>& Config::paramet {"lambdarank_norm", {}}, {"label_gain", {}}, {"lambdarank_position_bias_regularizer", {}}, + {"position_filename", {}}, {"metric", {"metrics", "metric_types"}}, {"metric_freq", {"output_freq"}}, {"is_provide_training_metric", {"training_metric", "is_training_metric", "train_metric"}}, @@ -1042,6 +1047,7 @@ const std::unordered_map& Config::ParameterTypes() { {"lambdarank_norm", "bool"}, {"label_gain", "vector"}, {"lambdarank_position_bias_regularizer", "double"}, + {"position_filename", "string"}, {"metric", "vector"}, {"metric_freq", "int"}, {"is_provide_training_metric", "bool"}, diff --git a/src/io/dataset_loader.cpp b/src/io/dataset_loader.cpp index 354936cfb01a..4dfc6e1dab24 100644 --- a/src/io/dataset_loader.cpp +++ b/src/io/dataset_loader.cpp @@ -225,7 +225,7 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac } dataset->data_filename_ = filename; dataset->label_idx_ = label_idx_; - dataset->metadata_.Init(filename); + dataset->metadata_.Init(filename, config_.position_filename.c_str()); if (!config_.two_round) { // read data to memory auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, rank, num_machines, &num_global_data, &used_data_indices); @@ -312,7 +312,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename, } dataset->data_filename_ = filename; dataset->label_idx_ = label_idx_; - dataset->metadata_.Init(filename); + dataset->metadata_.Init(filename, config_.position_filename.c_str()); if (!config_.two_round) { // read data in memory auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, 0, 1, &num_global_data, &used_data_indices); diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 9e82c9cc4702..c4e1a59ce0ec 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -21,13 +21,15 @@ Metadata::Metadata() { position_load_from_file_ = false; query_load_from_file_ = false; init_score_load_from_file_ = false; + position_filename_ = ""; #ifdef USE_CUDA cuda_metadata_ = nullptr; #endif // USE_CUDA } -void Metadata::Init(const char* data_filename) { +void Metadata::Init(const char* data_filename, const char* position_filename) { data_filename_ = data_filename; + position_filename_ = position_filename; // for lambdarank, it needs query data for partition data in distributed learning LoadQueryBoundaries(); LoadWeights(); @@ -595,12 +597,13 @@ void Metadata::LoadWeights() { void Metadata::LoadPositions() { num_positions_ = 0; - std::string position_filename(data_filename_); - // default position file name - position_filename.append(".position"); - TextReader reader(position_filename.c_str(), false); + if (position_filename_ == std::string("")) { + return; + } + TextReader reader(position_filename_.c_str(), false); reader.ReadAllLines(); if (reader.Lines().empty()) { + Log::Warning("Position file %s is not found.", position_filename_.c_str()); return; } Log::Info("Loading positions..."); From de43fc8d076f8c8e68474834ab368361f04a1f72 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:56:55 -0700 Subject: [PATCH 65/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index db42216fa561..c3d55b2092f0 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -81,7 +81,7 @@ Recommendations for gcc Users (MinGW, \*nix) Support for Position Bias Treatment ------------------------------------ -Often the relevance labels provided in Learning-to-Rank taks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user. +Often the relevance labels provided in Learning-to-Rank tasks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user. LightGBM uses an additional file to store positional data, like the following: :: From 626fa16f216d9a7a7f87faac37984242298cce61 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:57:14 -0700 Subject: [PATCH 66/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index c3d55b2092f0..791ce87306bf 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -92,7 +92,7 @@ LightGBM uses an additional file to store positional data, like the following: ... It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. -A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consiting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). +A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consisting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). The position file corresponds with training data file line by line, and has one position per line. From adcde0c0da0f697452937cfae7aa36b68658bbd7 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:02:45 -0700 Subject: [PATCH 67/97] Update Advanced-Topics.rst --- docs/Advanced-Topics.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 791ce87306bf..838060265d0c 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -82,7 +82,7 @@ Support for Position Bias Treatment ------------------------------------ Often the relevance labels provided in Learning-to-Rank tasks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user. -LightGBM uses an additional file to store positional data, like the following: +LightGBM can make use of positional data, for example provided in an additional file like the following: :: @@ -99,8 +99,6 @@ The position file corresponds with training data file line by line, and has one And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. In this case, LightGBM will load the position file automatically if it exists. -Also, you can include position column in your data file. Please refer to the ``position_column`` `parameter <#position_column>`__. - Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank `_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems `_, `A General Framework for Debiasing in CTR Prediction `_). From f3c338728e54059548f9c482f390a8b56b074065 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:04:46 -0700 Subject: [PATCH 68/97] Update src/objective/rank_objective.hpp Co-authored-by: James Lamb --- src/objective/rank_objective.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 2ce9c736c74b..5846e1b0092c 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -325,7 +325,7 @@ class LambdarankNDCG : public RankingObjective { // L2 regularization on position bias factors bias_first_derivative -= pos_biases_[i] * position_bias_regularizer_ * instance_count; bias_second_derivative -= position_bias_regularizer_ * instance_count; - // do Newton-Rhapson step to update position bias factors + // do Newton-Raphson step to update position bias factors pos_biases_[i] += learning_rate_ * bias_first_derivative / (std::abs(bias_second_derivative) + 0.001); } LogDebugPositionBiasFactors(); From e3c5e6f715438730a1414a928c9b51e9d7dd7ca7 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:07:35 -0700 Subject: [PATCH 69/97] Update src/io/metadata.cpp Co-authored-by: James Lamb --- src/io/metadata.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index c4e1a59ce0ec..74ac988dca99 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -603,8 +603,7 @@ void Metadata::LoadPositions() { TextReader reader(position_filename_.c_str(), false); reader.ReadAllLines(); if (reader.Lines().empty()) { - Log::Warning("Position file %s is not found.", position_filename_.c_str()); - return; + Log::Fatal("Position file '%s' is not found.", position_filename_.c_str()); } Log::Info("Loading positions..."); num_positions_ = static_cast(reader.Lines().size()); From d7376809788c254b496eb33a3e0580f644140156 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:26:10 -0700 Subject: [PATCH 70/97] Update metadata.cpp --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 74ac988dca99..94771f491485 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -596,10 +596,10 @@ void Metadata::LoadWeights() { } void Metadata::LoadPositions() { - num_positions_ = 0; if (position_filename_ == std::string("")) { return; } + num_positions_ = 0; TextReader reader(position_filename_.c_str(), false); reader.ReadAllLines(); if (reader.Lines().empty()) { From 5d836ed466928e87a0386b3d1e99d6012a91ecf3 Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Fri, 4 Aug 2023 20:13:23 +0800 Subject: [PATCH 71/97] Update python-package/lightgbm/basic.py Co-authored-by: James Lamb --- python-package/lightgbm/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 120ea353bb95..db4209c9aac5 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -1507,7 +1507,7 @@ def __init__( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. - position : list, numpy 1-D array, pandas Series or None, optional (default=None) + position : numpy 1-D array, pandas Series or None, optional (default=None) Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. From 1c6986254c05b6fcd6cd946affc9ef5e0feba7df Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Fri, 4 Aug 2023 20:16:58 +0800 Subject: [PATCH 72/97] Update python-package/lightgbm/basic.py Co-authored-by: James Lamb --- python-package/lightgbm/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index db4209c9aac5..ea70fe331e53 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -2262,7 +2262,7 @@ def create_valid( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. - position : list, numpy 1-D array, pandas Series or None, optional (default=None) + position : numpy 1-D array, pandas Series or None, optional (default=None) Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. From 5232cb870518bf060ce7a2b1305315209def70ae Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Fri, 4 Aug 2023 20:17:25 +0800 Subject: [PATCH 73/97] Update python-package/lightgbm/basic.py Co-authored-by: James Lamb --- python-package/lightgbm/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index ea70fe331e53..15285c0bf761 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -2711,7 +2711,7 @@ def set_position( Parameters ---------- - position : list, numpy 1-D array, pandas Series or None, optional (default=None) + position : numpy 1-D array, pandas Series or None, optional (default=None) Position of items used in unbiased learning-to-rank task. Returns From 7d9f0bbaa7a7cd3cd10fde4894b7b97945c0fedb Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Fri, 4 Aug 2023 20:17:57 +0800 Subject: [PATCH 74/97] Update python-package/lightgbm/basic.py Co-authored-by: James Lamb --- python-package/lightgbm/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 15285c0bf761..4337c8d009ba 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -2856,7 +2856,7 @@ def get_position(self) -> Optional[np.ndarray]: Returns ------- - position : list, numpy 1-D array, pandas Series or None, optional (default=None) + position : numpy 1-D array or None Position of items used in unbiased learning-to-rank task. """ if self.position is None: From 14c9c60ad5ef4d35d97a6f88c6e1be6b055a7096 Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Fri, 4 Aug 2023 20:24:19 +0800 Subject: [PATCH 75/97] Update src/io/metadata.cpp Co-authored-by: James Lamb --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 94771f491485..05444db88e7a 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -550,7 +550,7 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { for (int position : positions_) { position_set.insert(position); } - Log::Warning("position_set.size() = %ld", position_set.size()); + Log::Debug("number of unique positions found = %ld", position_set.size()); for (int position : position_set) { position_ids_.push_back(std::to_string(position)); } From 3a2031a8678ea32383d33e20fa4e74c2402a912d Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Fri, 4 Aug 2023 12:28:05 +0000 Subject: [PATCH 76/97] more infomrative fatal message address more comments --- python-package/lightgbm/basic.py | 13 +++++++------ src/io/metadata.cpp | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 120ea353bb95..67068a1ecf7d 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -1796,12 +1796,12 @@ def _lazy_init( reference: Optional["Dataset"], weight: Optional[_LGBM_WeightType], group: Optional[_LGBM_GroupType], - position: Optional[_LGBM_PositionType], init_score: Optional[_LGBM_InitScoreType], predictor: Optional[_InnerPredictor], feature_name: _LGBM_FeatureNameConfiguration, categorical_feature: _LGBM_CategoricalFeatureConfiguration, - params: Optional[Dict[str, Any]] + params: Optional[Dict[str, Any]], + position: Optional[_LGBM_PositionType] ) -> "Dataset": if data is None: self._handle = None @@ -2227,9 +2227,10 @@ def construct(self) -> "Dataset": else: # create train self._lazy_init(data=self.data, label=self.label, reference=None, - weight=self.weight, group=self.group, position=self.position, + weight=self.weight, group=self.group, init_score=self.init_score, predictor=self._predictor, - feature_name=self.feature_name, categorical_feature=self.categorical_feature, params=self.params) + feature_name=self.feature_name, categorical_feature=self.categorical_feature, + params=self.params, position=self.position) if self.free_raw_data: self.data = None self.feature_name = self.get_feature_name() @@ -2241,9 +2242,9 @@ def create_valid( label: Optional[_LGBM_LabelType] = None, weight: Optional[_LGBM_WeightType] = None, group: Optional[_LGBM_GroupType] = None, - position: Optional[_LGBM_PositionType] = None, init_score: Optional[_LGBM_InitScoreType] = None, - params: Optional[Dict[str, Any]] = None + params: Optional[Dict[str, Any]] = None, + position: Optional[_LGBM_PositionType] = None ) -> "Dataset": """Create validation data align with current Dataset. diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 94771f491485..090af7ce8f4f 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -529,16 +529,16 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { num_positions_ = 0; return; } + #ifdef USE_CUDA + Log::Fatal("Positions in learning to rank is not supported in CUDA version yet."); + #endif // USE_CUDA if (num_data_ != len) { - Log::Fatal("Length of positions is not same with #data"); + Log::Fatal("Positions size (%i) doesn't match data size (%i)", len, num_data_); } if (positions_.empty()) { positions_.resize(num_data_); } num_positions_ = num_data_; position_load_from_file_ = false; - #ifdef USE_CUDA - Log::Fatal("Positions in learning to rank is not supported in CUDA version yet."); - #endif // USE_CUDA #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) for (data_size_t i = 0; i < num_positions_; ++i) { From 757f7cb5b19134394d47dec792b29cfb3b225223 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Fri, 4 Aug 2023 12:36:32 +0000 Subject: [PATCH 77/97] update documentation for more flexible position specification --- docs/Advanced-Topics.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 838060265d0c..58a8afcf72bd 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -94,10 +94,7 @@ LightGBM can make use of positional data, for example provided in an additional It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consisting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). -The position file corresponds with training data file line by line, and has one position per line. - -And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. -In this case, LightGBM will load the position file automatically if it exists. +The position file corresponds with training data file line by line, and has one position per line. The position file path can be specified using the parameter ``position_filename``, or through the ``Dataset`` constructor when using Python API. Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. From 70fc1919f9d40786e448e46d94e4430e8f6f5539 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Fri, 4 Aug 2023 13:35:36 +0000 Subject: [PATCH 78/97] fix SetPosition add tests for get_position and set_position --- python-package/lightgbm/basic.py | 8 +- src/io/metadata.cpp | 22 +- tests/c_api_test/preb.txt | 500 +++++++++++ tests/python_package_test/data.csv | 100 +++ tests/python_package_test/data_dask.csv | 1000 ++++++++++++++++++++++ tests/python_package_test/test_engine.py | 15 + 6 files changed, 1631 insertions(+), 14 deletions(-) create mode 100644 tests/c_api_test/preb.txt create mode 100644 tests/python_package_test/data.csv create mode 100644 tests/python_package_test/data_dask.csv diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 79ba0128f7a4..85b0c7d881c6 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -1507,8 +1507,6 @@ def __init__( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. - position : numpy 1-D array, pandas Series or None, optional (default=None) - Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. feature_name : list of str, or 'auto', optional (default="auto") @@ -1528,6 +1526,8 @@ def __init__( Other parameters for Dataset. free_raw_data : bool, optional (default=True) If True, raw data is freed after constructing inner Dataset. + position : numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. """ self._handle: Optional[_DatasetHandle] = None self.data = data @@ -2263,12 +2263,12 @@ def create_valid( sum(group) = n_samples. For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups, where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc. - position : numpy 1-D array, pandas Series or None, optional (default=None) - Position of items used in unbiased learning-to-rank task. init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None) Init score for Dataset. params : dict or None, optional (default=None) Other parameters for validation Dataset. + position : numpy 1-D array, pandas Series or None, optional (default=None) + Position of items used in unbiased learning-to-rank task. Returns ------- diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 85adfcd805ef..dad64125bbf8 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -540,19 +540,21 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { position_load_from_file_ = false; - #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) + position_ids_.clear(); + std::unordered_map map_id2pos; for (data_size_t i = 0; i < num_positions_; ++i) { - positions_[i] = positions[i]; + if (map_id2pos.count(positions[i]) == 0) { + int pos = static_cast(map_id2pos.size()); + map_id2pos[positions[i]] = pos; + position_ids_.push_back(std::to_string(positions[i])); + } } - position_ids_.clear(); - std::set position_set; - for (int position : positions_) { - position_set.insert(position); - } - Log::Debug("number of unique positions found = %ld", position_set.size()); - for (int position : position_set) { - position_ids_.push_back(std::to_string(position)); + Log::Debug("number of unique positions found = %ld", position_ids_.size()); + + #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) + for (data_size_t i = 0; i < num_positions_; ++i) { + positions_[i] = map_id2pos.at(positions[i]); } } diff --git a/tests/c_api_test/preb.txt b/tests/c_api_test/preb.txt new file mode 100644 index 000000000000..f35c133463e6 --- /dev/null +++ b/tests/c_api_test/preb.txt @@ -0,0 +1,500 @@ +0.72816922538206474 +0.34254413558637198 +0.39375878909343448 +0.53665466907657056 +0.38072398950936409 +0.34017648118647142 +0.45978203210567342 +0.44628065737977557 +0.60080289251541041 +0.53744413394458634 +0.42064207683507732 +0.60885892984431689 +0.56631892989602972 +0.59384597288021679 +0.41509342657650794 +0.6447246230140683 +0.41398638532950588 +0.49619015828026219 +0.50816544640733008 +0.5433913020081389 +0.70765788333666124 +0.58737495344973312 +0.55202925139748704 +0.49384895389338523 +0.52715450986408574 +0.62003028099221735 +0.5271304222108627 +0.61143126453405039 +0.51040226321315518 +0.53516911727194239 +0.58265400049973981 +0.60839584467348884 +0.32045717477916147 +0.50436461383302911 +0.56626759697004803 +0.42529703085324583 +0.39216853111023542 +0.31062337068447893 +0.39695081862302423 +0.56257576053762914 +0.21787838987724456 +0.51317474641799277 +0.43137842267424936 +0.5632093944281632 +0.41152383935988024 +0.18558569537594216 +0.56585961941753482 +0.48065327283132342 +0.74050687924153469 +0.37556583740692273 +0.82519659752152996 +0.5302330114162398 +0.22432282431040096 +0.66954005712518405 +0.62112099036121549 +0.52787120676714061 +0.32934423532118201 +0.37602098166850595 +0.43298291586822835 +0.52925150429120693 +0.46250078955240304 +0.60757346403046508 +0.74149872758318369 +0.44285821827108729 +0.67928949090859736 +0.43382497854499585 +0.51231584748043701 +0.247140869791797 +0.7226842943309939 +0.57036909154325477 +0.41340599994509702 +0.5685276873486278 +0.71280318144675847 +0.38721005503175732 +0.71966468278946993 +0.76854678984409386 +0.48834863757820024 +0.40267567760863782 +0.70338647717389102 +0.61775482428378359 +0.41428705286435008 +0.56361320875176002 +0.6442020838419894 +0.70651580109729517 +0.20778472573516621 +0.41478764955120984 +0.48078639190418138 +0.59362686094208328 +0.44758202367995031 +0.64385479183149297 +0.35720388319190521 +0.35754670393997789 +0.49232989003411853 +0.56636920108340971 +0.48569817022544198 +0.63277119005777882 +0.43393064976357837 +0.43556665306809156 +0.7331161120590588 +0.27973037887813385 +0.45416195308273949 +0.43089514054677458 +0.54906154859737255 +0.36168897508855047 +0.47064286495658053 +0.67113198006622732 +0.5362421344673165 +0.57324619386279518 +0.34414679089845102 +0.52348226159323219 +0.41676568695820876 +0.49108911497066665 +0.44330930549031294 +0.24876192766094601 +0.58714982657373627 +0.71013870537805623 +0.34397499263868941 +0.57155232362819841 +0.41048575310259899 +0.49737430255200454 +0.47189096802485031 +0.66764025079844513 +0.52923772808164837 +0.46340444214827337 +0.35608724781996315 +0.46420421801911049 +0.51668476760268423 +0.64831275301692892 +0.43918091422652927 +0.48277245420321302 +0.55849430746574891 +0.47611787420285684 +0.48562808680585734 +0.43264795894625369 +0.38066444735962551 +0.67075756760210925 +0.41051782077733057 +0.67536163223484957 +0.46664307460700077 +0.52459413309980529 +0.4746240504700428 +0.58405071264679709 +0.52499392446764903 +0.50962484356825899 +0.55937379802764442 +0.35217297281347715 +0.38315020415159362 +0.48029668437134981 +0.67027252713953178 +0.42114818018884226 +0.33896134332400352 +0.66532937236872891 +0.50027400566434232 +0.41849942912247773 +0.53283022597370455 +0.37819833075377918 +0.42174948103710375 +0.47016465206864677 +0.48970903509423114 +0.42453311555017942 +0.73636490994064541 +0.52010828356038052 +0.64082625083353251 +0.22300684912301558 +0.54468162436264855 +0.45938588802016334 +0.60439824927919816 +0.64378129238701565 +0.36465235254606332 +0.68695643355221081 +0.74540016978378254 +0.51080984365545212 +0.43131123272188487 +0.6896752324217279 +0.47581100181976643 +0.66577981888510618 +0.49983121317630602 +0.40372208440824964 +0.35920836556409236 +0.18236290456819801 +0.24263318676107182 +0.48323045259130437 +0.24094872612509108 +0.48035420002415574 +0.37747678847086152 +0.73672532220921483 +0.50478964632958079 +0.46308123007068225 +0.72593723350992023 +0.50683500485100796 +0.53762469363299326 +0.4176841526900435 +0.55185622044794536 +0.47546001640382751 +0.5522531876403215 +0.59346320905690964 +0.45915342031396184 +0.46100132693087015 +0.30310709558752225 +0.52849719589140509 +0.43300081514979633 +0.27302253150363232 +0.46146687612990839 +0.49419972338446916 +0.53541256633700318 +0.28231826885884193 +0.54800509322342883 +0.46092780871857986 +0.65946375879983188 +0.30311257538621356 +0.32448644717174041 +0.60602844828214997 +0.5266144627430438 +0.42260572901779836 +0.41973578749864188 +0.43773105150144687 +0.65203538865860078 +0.5190971064815445 +0.46870737786818895 +0.37819632742074116 +0.64222288958825147 +0.29285772401646853 +0.5782993574365668 +0.50816383026400025 +0.40459310252704089 +0.72451394301377325 +0.3988911605759109 +0.38948876687559641 +0.58786635655785735 +0.4567731124678342 +0.52254990688142922 +0.342216292004832 +0.41025189598037087 +0.60922959301919655 +0.49801830351262411 +0.45509521650260859 +0.61839273426823838 +0.51948512079115539 +0.51957016625028241 +0.54859674358076826 +0.49775636069388524 +0.28559977239767326 +0.22559120432624344 +0.51533796680029975 +0.43457110449375641 +0.32310354449152678 +0.3470205901517695 +0.66162630391137311 +0.41159419987099966 +0.35347673353871789 +0.37267078033390405 +0.43309743695371639 +0.38197279548033053 +0.67686196129957121 +0.67492684647667178 +0.56623301854706309 +0.44958559907571466 +0.66507670329237445 +0.46754769110307898 +0.46277629753232408 +0.46278185023237489 +0.49261676851939318 +0.32840852227458994 +0.55009590938324393 +0.55740327930252231 +0.60208788647637956 +0.59694564464334576 +0.37164726510937918 +0.55942019678447774 +0.19967172616498283 +0.32230119630976839 +0.48031834660795408 +0.37105438680830116 +0.49448349233626626 +0.69982993678588823 +0.61657392311280412 +0.52721190658957773 +0.71731539988068571 +0.37359486367193034 +0.41351991255222226 +0.43421480490214148 +0.51422498200331535 +0.30111836867200181 +0.41756836465088154 +0.48635495811950902 +0.3983806388716683 +0.46676700527201159 +0.42954972869201297 +0.38207465958363551 +0.54058439311800122 +0.53482361057077665 +0.44111059747635689 +0.34302590111088133 +0.66230380938398514 +0.28452505834512065 +0.40382652376986966 +0.40473071866671301 +0.40948918067824103 +0.41710643416750176 +0.32732661890008058 +0.39110854415326002 +0.42963519644513043 +0.48061784046160183 +0.61922246525791003 +0.35928763954401505 +0.66598022986198879 +0.39155974102201996 +0.72525893075604475 +0.53880138638651998 +0.5336261241365855 +0.65057732717760763 +0.50749372436514861 +0.43631033184402251 +0.73423630368236259 +0.49201633161964392 +0.61920314297939205 +0.57285661477171756 +0.51855006294612416 +0.42934983013423655 +0.40543433340051949 +0.53198765341485688 +0.47542585769879675 +0.52745815798800433 +0.57114368037229968 +0.59534514535110339 +0.44005221197410377 +0.44939601527291062 +0.44826114468524919 +0.4323617741571773 +0.458566154799254 +0.47314581804815653 +0.58744189351626297 +0.63675203666532387 +0.5804116726857933 +0.24756701756211835 +0.41454919412031699 +0.50434233050585342 +0.44163240481900445 +0.7015347144176427 +0.37889569458155642 +0.58278502159514001 +0.34447989922720518 +0.53332757736827097 +0.4233394870228937 +0.71945766050826132 +0.34937745895084515 +0.51547576377624804 +0.6729458724709223 +0.43721697013092459 +0.51293251676173734 +0.47611565769859765 +0.66858319468077898 +0.56444042514485959 +0.3232993461576506 +0.5336272685400133 +0.38312680818919731 +0.46833368528524411 +0.42722984600517683 +0.76493746129395712 +0.42153961662322126 +0.41681766033430345 +0.43929719874856737 +0.69500378552300468 +0.38928035790199378 +0.43147025074078554 +0.55920452412531474 +0.42894793416255833 +0.43442162935635986 +0.503212059848682 +0.68012930263242488 +0.48914064710841854 +0.27612129532323215 +0.53431245370499159 +0.6045973005562707 +0.63649103198737655 +0.50529097096682396 +0.58855254626783615 +0.60332172849418841 +0.60948433205877073 +0.21033368146558756 +0.40105860535592164 +0.76576952226902295 +0.56032901397782431 +0.49716119079344812 +0.54671301465650146 +0.50339186720895035 +0.57222958060106743 +0.53092125118898159 +0.47977652816344601 +0.68804390053570019 +0.67587717113717383 +0.45151198485657995 +0.58903092289861969 +0.41270368434201915 +0.72868809400599199 +0.48976693927324211 +0.46315791511471843 +0.69203149057457658 +0.69066004578091489 +0.22752843011573498 +0.44746859510224962 +0.56300744465883035 +0.57267158573149934 +0.53039800754269206 +0.53381382719534376 +0.66539039062918004 +0.56105480141003827 +0.59681767781266593 +0.46630916540536593 +0.39569253482998978 +0.64214418716534494 +0.41401841078073093 +0.41803364939949533 +0.66694748698812323 +0.45533672480245041 +0.47959617759024908 +0.78622928107673162 +0.59044680460965016 +0.46270650723991447 +0.52966423011216834 +0.59309508150314827 +0.45487655762459989 +0.44987884334688288 +0.69559467323198632 +0.37758323022318574 +0.46928051850278013 +0.48270761970812393 +0.59196315446282577 +0.40657323467155099 +0.4060445606251889 +0.34813812777382963 +0.67773822398445416 +0.54567747328268879 +0.71457208044119724 +0.52993603546746437 +0.50745178395133106 +0.43442266648832961 +0.46072147155862792 +0.53823902515603284 +0.7552831359895138 +0.46753380199833788 +0.44249910937574538 +0.43451356469444918 +0.48287308211005392 +0.6834407324164935 +0.5871760108194507 +0.46838801465223917 +0.69859760734015108 +0.46225488812609877 +0.70602904909814235 +0.25980202402639374 +0.5151635985827917 +0.5079515548240946 +0.49757674154255982 +0.45555838090674267 +0.42074825662008747 +0.79027206514030135 +0.47253419563245308 +0.3793242141395845 +0.50481275780964652 +0.43369027374583646 +0.42404092819642814 +0.39147593198624142 +0.48342051831462446 +0.32219711323686961 +0.5942662336530895 +0.62387595564165399 +0.45508434052140312 +0.45190156205406901 +0.44650683617222847 +0.32343032516157727 +0.44318145059549174 +0.54742356980393136 +0.56828808304948708 +0.56799254528844034 +0.59471453737286029 +0.46580382092460726 +0.55244497014089167 +0.54880384388723835 +0.493056760399538 +0.56765364950637343 +0.55994378859402938 +0.55320906145280446 +0.58838693902959793 +0.55917435186993925 +0.6504168677181329 +0.43055030187935561 +0.5076695055093613 +0.60108248996680347 +0.55195483098733944 +0.47746471327886564 +0.44391354440749742 +0.48833752671540848 +0.36657132142256094 +0.68656415852126007 +0.71930502856513934 +0.39245119176277071 +0.550361588643836 +0.41242219282318993 +0.49308951662419243 diff --git a/tests/python_package_test/data.csv b/tests/python_package_test/data.csv new file mode 100644 index 000000000000..db7d1d702d9c --- /dev/null +++ b/tests/python_package_test/data.csv @@ -0,0 +1,100 @@ +8.648289,-1.918771,-0.026514,-0.074446,0.257550 +22.812042,1.586017,-1.237815,0.662131,0.113517 +116.259508,-1.519370,-0.484234,1.032465,2.122156 +-41.296264,0.813517,-1.230864,-0.322062,-0.783253 +-44.592294,1.632411,-1.430141,-1.247783,-0.252568 +-74.928338,1.158111,0.791663,-1.214189,-1.006017 +41.292330,-2.619745,0.821903,1.564644,-0.035826 +84.306971,-1.260884,0.917862,0.404982,1.765454 +-32.890778,-0.601707,1.852278,-0.291694,-0.600639 +-27.378431,0.341756,1.876171,0.150394,-0.759133 +-40.078349,-3.241267,-1.024388,-0.059525,-0.926930 +-19.549355,-1.150994,0.375698,0.110923,-0.544383 +-29.478456,0.822545,-1.220844,-1.057711,-0.013497 +-14.679364,1.441273,-1.435862,0.130741,-0.440044 +19.421082,0.005113,-0.234587,0.261055,0.296120 +-24.953130,-1.106335,-1.196207,-0.185659,-0.479174 +51.248443,0.822060,1.896793,0.412781,0.963376 +44.964201,0.686260,-1.612716,-1.867265,2.314659 +-33.948808,0.324084,-0.385082,-1.763040,0.343618 +-47.256284,0.915402,0.328751,-0.501757,-0.808494 +-4.692470,1.179440,-0.469176,1.831459,-1.320233 +-25.178353,0.173181,0.385317,-1.245739,0.214094 +45.718943,0.812862,0.629629,0.307300,0.899600 +-4.564190,0.091761,-1.987569,-0.299007,0.087047 +-34.048333,-0.114540,1.237816,1.353872,-1.713135 +7.950252,1.402794,-1.401851,-0.909387,0.791032 +-82.789212,0.515035,0.513786,-0.937825,-1.377669 +45.173689,-1.607483,0.184634,1.307143,0.227460 +-44.914698,-1.328186,0.196861,-1.959670,0.208864 +-15.369559,0.975120,-0.147057,-0.677162,0.075805 +69.845248,-0.192361,0.301547,2.463242,0.060230 +56.481809,1.305479,0.021004,0.813510,0.825416 +-11.091510,-0.730367,0.216459,0.679598,-0.715304 +-18.912247,1.158596,-0.820682,0.787085,-0.974682 +90.350893,-1.191303,0.656554,0.473833,1.865775 +-38.081046,-0.392108,-1.463515,-0.327662,-0.702053 +-9.631647,-1.550663,0.068563,-0.503476,0.099651 +-30.786857,-0.889514,-0.815810,-0.753736,-0.245388 +4.356590,1.886186,0.174578,0.404051,-0.161286 +-8.660092,0.322719,-0.827231,-0.288659,-0.018513 +-78.909071,-0.661786,0.852433,0.186454,-2.025143 +-2.636663,0.361396,1.538037,-0.645120,0.361636 +19.805827,0.324166,-0.130143,-0.310267,0.681953 +2.518608,-1.070892,0.482472,-0.857158,0.625667 +-8.152970,0.069802,-0.385314,-0.600217,0.199060 +29.770925,0.250493,0.346448,0.296985,0.521942 +-27.444725,0.244967,-0.506943,0.243687,-0.822220 +35.318364,-0.115648,-0.301104,0.171368,0.738467 +-16.114609,1.579213,0.767435,-0.234137,-0.234153 +-11.356382,1.031000,0.931280,0.611676,-0.676922 +0.651292,1.477894,-0.518270,0.357113,-0.219672 +31.204561,-0.019016,-1.002529,-0.159939,0.857660 +-43.266465,0.331263,0.975545,-0.309212,-0.839218 +63.449672,-0.108760,0.401712,1.532739,0.519347 +10.251606,0.473238,-0.072829,0.714000,-0.223463 +-76.543649,-0.446515,0.856399,-1.514847,-0.846794 +-43.030150,0.412931,-0.563725,-0.321386,-0.825497 +10.196025,0.064280,-1.077745,1.088951,-0.471932 +-15.924806,2.143944,0.633919,-0.651600,0.045572 +-36.012281,0.504987,0.865755,-0.114736,-0.792521 +-33.392396,1.142823,0.751933,-1.168678,-0.034712 +86.423410,2.153182,-0.767348,0.959271,1.451144 +6.129007,0.276691,0.827183,0.341152,-0.077102 +-51.221118,-2.123896,-0.525755,-0.599393,-0.839722 +-49.704076,0.747294,0.610370,-0.560181,-0.828995 +-81.013025,-0.460639,1.057122,-0.719844,-1.478522 +-32.461177,0.058209,-1.142970,0.153725,-0.883857 +-17.082839,0.024510,0.497998,-0.773010,0.097676 +57.383351,-0.315269,0.758969,0.651391,0.954002 +-13.195178,-1.448084,-1.407464,0.232050,-0.471038 +-70.213079,-0.342715,-0.802277,-0.420645,-1.415371 +-21.858771,0.293072,-0.714351,0.232254,-0.680025 +35.110923,-0.151785,0.588317,-1.952088,2.133033 +-31.118078,-0.919424,1.549934,0.473592,-1.062304 +84.232583,-0.990536,-0.566298,2.190456,0.586857 +33.205762,0.443819,0.774634,-0.707669,1.266911 +-58.936136,-0.474945,-0.653329,-0.334501,-1.200296 +-42.273368,-1.724918,-0.562288,-1.913280,0.241962 +2.340775,1.277665,-0.591571,0.117327,-0.020902 +40.279993,-0.264657,2.720169,1.453534,0.013002 +30.173963,1.083051,1.053802,0.560785,0.357787 +16.824428,0.647689,1.523030,-0.138264,0.496714 +-5.368955,-0.208122,-0.493001,-0.622700,0.280992 +17.166576,-0.217681,1.098777,-0.202193,0.547097 +70.785717,-0.072010,1.003533,1.356240,0.812526 +43.069118,-0.012247,-0.897254,0.628346,0.624120 +-35.638264,0.310908,1.475356,-0.213447,-0.718444 +-1.219277,0.357015,-0.692910,0.849602,-0.589365 +32.156035,-1.236951,-1.320457,0.781823,0.259883 +23.652480,-0.898415,0.491919,-0.576904,0.950424 +-7.942392,0.097078,0.968645,0.513267,-0.529760 +126.701586,0.570891,1.135566,3.852731,0.515048 +-4.640738,-0.463418,-0.465730,0.542560,-0.469474 +-33.422466,-0.908024,-1.412304,0.314247,-1.012831 +41.198261,2.189803,-0.808298,0.183342,0.872321 +48.530080,-0.981509,0.462103,0.010233,1.163164 +20.295596,-0.818221,2.092387,0.595157,0.096996 +17.658867,0.224092,0.012592,-0.401220,0.690144 +54.625086,0.067528,-1.424748,-0.225776,1.465649 +-38.533085,-0.485364,0.081874,-0.236819,-0.772825 diff --git a/tests/python_package_test/data_dask.csv b/tests/python_package_test/data_dask.csv new file mode 100644 index 000000000000..83e2eb959c32 --- /dev/null +++ b/tests/python_package_test/data_dask.csv @@ -0,0 +1,1000 @@ +56.738133,1.239584,0.604121,1.068379,-0.396230 +-6.159212,-0.164335,-1.015879,-0.993359,0.212093 +25.505362,0.493318,0.700310,-0.858358,0.184836 +-15.630020,-0.476221,0.193590,0.262561,0.874389 +-21.310863,-0.529760,0.968645,0.097078,0.513267 +-51.704342,-1.104863,-0.513196,1.217959,0.220541 +-82.951301,-1.692957,-1.103589,-0.988591,-0.098340 +-8.817806,-0.026521,-0.744903,-0.163067,-0.881875 +-77.858664,-1.478522,1.057122,-0.460639,-0.719844 +-64.662621,-1.175595,-0.948348,0.597228,-0.895070 +-29.052979,-0.562168,-0.805870,-1.683438,-0.209222 +-63.495550,-1.534114,-0.748487,0.332314,1.277677 +40.551412,0.823171,-1.295079,-1.289961,0.073318 +-6.366368,-0.185846,1.218971,-0.056588,0.309999 +-65.837724,-1.097368,1.191899,-0.739371,-1.476925 +5.133149,0.326133,-0.184902,0.924027,-1.251114 +1.896450,0.257753,-0.155259,0.334176,-1.241761 +-36.887882,-0.302181,-0.064184,-0.361311,-2.603137 +-20.211424,-0.649373,-0.600424,1.419603,1.321304 +27.517167,0.664703,-0.497571,1.001825,-0.552921 +-13.287606,-0.339122,0.457806,-0.123733,0.370061 +-106.217377,-2.087027,-0.909683,-0.031059,-0.584618 +-15.598486,-0.103431,1.594442,2.095844,-1.239053 +-1.023335,0.159856,-0.866175,1.265708,-1.027675 +20.523607,0.342725,0.447709,0.569767,0.456753 +-4.887067,0.032797,-0.924233,-0.230401,-0.758495 +-3.055117,-0.039307,1.431367,0.333860,-0.134497 +37.239465,0.505589,-0.404397,2.271450,1.489113 +17.744286,0.189582,0.677875,-2.703232,1.001046 +-47.479018,-1.062304,1.549934,-0.919424,0.473592 +72.906480,1.593187,-0.125787,-0.989605,-0.511216 +-13.025066,-0.110665,-0.369207,0.841984,-0.896642 +-23.193862,-0.517611,1.188393,-0.016423,0.223788 +-37.381345,-0.727137,0.620672,-0.074433,-0.247519 +-3.603126,0.077368,0.538910,1.523124,-0.861284 +121.397723,2.426716,-0.564774,-1.377618,0.432960 +100.082249,1.873298,1.281016,-0.447322,1.080048 +2.983104,0.195025,0.622711,-1.130615,-0.758282 +110.804925,2.184097,-0.611385,-0.580648,0.570507 +-69.869577,-1.531108,-0.062191,0.572057,0.514255 +62.613652,1.124113,0.407052,-0.772878,0.947526 +-30.282812,-0.673491,2.145149,-0.835347,0.278994 +7.946695,-0.089736,1.800940,-0.676392,1.440117 +30.950050,0.444198,1.346226,0.466545,1.101323 +-43.659691,-1.189412,0.638392,-0.900621,1.642673 +51.905761,1.180641,0.051198,0.045223,-0.627313 +26.187586,0.494030,-0.671623,-0.550305,0.260674 +-32.882334,-0.964705,1.590582,-0.284385,1.628469 +13.606900,0.453509,0.446873,0.058984,-0.982294 +-39.418565,-0.792521,0.865755,0.504987,-0.114736 +41.260883,1.081767,1.328933,0.622070,-1.312219 +-4.181337,-0.140375,1.352203,-0.524567,0.307613 +67.374861,1.021383,-1.879924,-1.585983,2.088443 +48.908565,0.881408,1.355443,-0.916274,0.721135 +41.936281,0.756789,-0.244080,-1.016683,0.612469 +24.979850,0.432263,-0.712174,-0.707626,0.470044 +30.884532,0.835692,1.441569,0.529804,-1.129707 +-71.527626,-1.713135,1.237816,-0.114540,1.353872 +65.401599,1.392465,0.260322,0.288694,-0.250047 +-96.630682,-2.025143,0.852433,-0.661786,0.186454 +-23.147713,-0.546244,0.183360,-1.478912,0.391804 +35.637195,0.624120,-0.897254,-0.012247,0.628346 +-4.760304,0.228996,0.298158,0.603248,-1.857901 +62.711268,1.296995,-0.504775,-0.999302,-0.022868 +-54.545646,-1.143726,-0.208117,-0.033230,0.108560 +-26.436378,-0.601368,0.440475,-1.592994,0.319782 +17.816529,0.409819,1.899882,0.672574,-0.241258 +47.170779,0.586857,-0.566298,-0.990536,2.190456 +4.204770,-0.015310,-0.973069,0.119580,0.579291 +39.632693,0.858451,-1.448591,0.038238,-0.234621 +0.533436,0.099651,0.068563,-1.550663,-0.503476 +-34.513746,-0.658116,-0.819258,-1.345871,-0.303726 +59.426661,1.200532,-1.117186,-1.967070,0.140360 +-76.786718,-2.135674,0.223239,1.056057,3.137749 +-33.969474,-0.513867,0.955142,-0.062679,-1.059214 +35.267866,0.538296,-0.839210,-0.364953,1.072507 +-7.417766,-0.021367,0.884045,-2.424240,-0.747212 +-53.186174,-1.222128,-0.374821,-0.240325,0.712998 +-41.555817,-0.883857,-1.142970,0.058209,0.153725 +-40.551873,-0.974682,-0.820682,1.158596,0.787085 +22.125167,0.585904,0.218534,-1.415267,-0.736770 +31.915628,0.183835,-1.004055,0.349800,2.693034 +-34.971223,-0.889241,1.472671,0.876047,0.955301 +-99.880047,-1.907808,1.887688,-0.413606,-0.860385 +77.754511,1.683928,-0.038508,1.078681,-0.458884 +1.385733,0.295624,-1.167780,1.476934,-1.516643 +-46.911212,-0.895346,0.651136,-0.995815,-0.408101 +48.488916,1.082691,1.325797,-0.093636,-0.471125 +7.271360,0.056650,0.486502,-0.070499,0.529693 +86.540739,2.024310,-0.661982,0.189706,-1.363174 +-51.802485,-1.111458,1.140149,0.498222,0.246505 +-94.183293,-2.211862,-0.266652,-1.423957,1.533434 +51.833880,0.954002,0.758969,-0.315269,0.651391 +-14.364838,-0.127918,0.203464,-1.606446,-0.955540 +44.351450,0.996571,-0.190503,0.640480,-0.466495 +92.510681,2.086047,0.416446,-0.361463,-1.014759 +18.485142,0.712712,0.437313,-0.372319,-1.883150 +-94.003921,-1.869742,0.449219,0.190424,-0.388521 +5.985189,0.088658,-0.316073,-0.617652,0.197316 +-15.919441,-0.278963,0.340051,0.790372,-0.279760 +54.775949,1.201214,-1.008086,-2.038125,-0.408075 +8.410255,0.338553,-1.025753,0.216625,-0.937926 +37.689252,0.672861,-0.573602,-0.354041,0.591814 +5.762199,0.022466,1.114322,-1.180813,0.547119 +49.781995,0.645484,0.219150,-0.307778,2.163255 +38.549010,0.742095,1.561511,1.301741,0.299293 +-19.511945,-0.149941,0.493655,-0.524662,-1.433152 +-35.857591,-0.614323,-1.371743,0.978890,-0.709789 +-47.358007,-0.654076,1.373659,0.511203,-1.830633 +-49.513008,-0.905732,1.374438,-0.595661,-0.653766 +12.278915,0.166452,2.455300,0.289169,0.492451 +17.728568,0.331980,-0.435486,0.709452,0.190500 +-36.095843,-0.935439,0.808058,-0.535963,1.085982 +-27.141358,-0.362441,1.160827,-1.294681,-1.119670 +-22.144920,-0.502975,-0.234408,0.294224,0.263487 +-32.826274,-0.539760,-0.978373,0.195845,-0.778305 +1.100857,-0.258796,-0.295480,0.560919,1.598647 +60.240622,1.238283,1.702215,0.308833,0.021272 +-58.487004,-0.924564,-0.213457,-0.327017,-1.597599 +48.844404,0.681891,-0.359292,0.583928,1.846707 +5.439839,-0.005596,-0.387100,-1.091701,0.668742 +27.968954,0.439501,1.674492,0.457773,0.778937 +55.403758,1.266911,0.774634,0.443819,-0.707669 +64.068543,1.065172,-0.836921,-0.916321,1.452617 +83.949823,1.897767,0.764030,-0.834323,-0.947893 +18.169315,0.101856,-1.467525,-1.239107,1.549020 +31.180192,0.782130,1.081985,-0.779853,-0.790899 +-15.189929,-0.313058,0.604515,-1.250408,-0.000709 +-89.009748,-1.448014,-0.502054,0.440014,-2.198806 +-15.403408,-0.385022,-2.130566,0.141257,0.382989 +-0.480558,0.435975,0.126178,0.682069,-2.532246 +27.728969,0.558327,-0.920674,0.538756,0.076005 +-126.684931,-2.471645,-0.203045,0.577072,-0.796895 +16.991130,0.386323,1.573020,1.755476,-0.204471 +70.432730,1.022570,0.563909,1.384273,2.439752 +33.505332,0.534506,1.795211,-0.320668,0.887655 +-5.830533,-0.045512,0.329509,-0.569833,-0.424236 +40.096885,0.850898,-0.103222,0.390465,-0.137372 +69.158404,1.465649,-1.424748,0.067528,-0.225776 +42.172756,0.662881,-1.296832,0.181022,1.173474 +43.176080,0.680216,0.319652,-1.778588,1.192508 +10.790567,0.463206,-1.232523,0.848174,-1.367138 +-95.070988,-2.153343,-0.862776,-0.478837,1.097153 +-19.340279,-0.238948,0.755391,-0.576771,-0.907564 +-61.478584,-1.029372,1.112688,-1.522359,-1.352670 +86.675134,1.923446,-0.471264,-1.689183,-0.774615 +-15.163206,0.025388,-0.689728,-0.013838,-1.919673 +87.515288,1.727543,0.120031,0.038003,0.436324 +-28.502093,-0.718832,-0.064188,-0.194697,0.745005 +123.716946,2.412615,-0.262891,-0.019260,0.784604 +-19.041372,-0.103255,1.661470,-0.175854,-1.643189 +30.861879,0.425490,-0.709441,-1.371674,1.197247 +-27.945422,-0.412998,-0.848429,-0.031439,-0.926698 +-27.424262,-0.641482,0.754291,0.800410,0.431923 +26.530196,0.482067,-1.927673,0.393797,0.368733 +75.369204,1.565524,1.881157,-0.555200,-0.065750 +17.285247,0.190636,0.397927,-1.032524,0.941311 +-87.930527,-1.970104,0.730764,-1.211172,0.892597 +10.574060,0.067856,-0.846357,0.484733,0.852774 +-30.314904,-0.822420,-0.009300,0.000207,1.121031 +-39.345829,-0.800039,-0.457302,1.237438,-0.063525 +8.310568,0.280992,-0.493001,-0.208122,-0.622700 +25.103271,0.012499,1.058729,-1.668599,2.868403 +-70.487875,-1.451176,0.363632,-1.252393,-0.012089 +35.199118,0.840644,-1.889541,-0.446183,-0.652624 +23.749863,0.202329,1.818062,-0.733033,1.631857 +23.670387,0.998010,-0.139590,2.088375,-2.896255 +40.012818,0.638660,0.148089,0.367620,1.058118 +22.669947,0.513106,0.615367,0.738810,-0.259547 +39.422200,0.886887,0.198948,-2.604214,-0.420762 +36.866459,0.874517,-1.042044,-1.203201,-0.649765 +48.831332,0.921802,0.606851,0.420094,0.482688 +-37.798006,-0.822220,-0.506943,0.244967,0.243687 +23.664217,0.629530,-0.631578,0.895355,-0.804316 +10.419259,0.257199,-0.325611,0.007880,-0.240665 +58.935380,0.892954,0.712322,-0.409687,1.829620 +3.090739,0.502745,1.629166,-0.965665,-2.493271 +22.195611,0.227460,0.184634,-1.607483,1.307143 +-59.681008,-1.085151,1.244680,2.949094,-0.825411 +-0.444153,-0.139397,1.318302,-1.909356,0.739653 +-6.553380,-0.092944,0.186416,1.116834,-0.239502 +-14.140045,-0.688150,-0.324831,0.981765,2.252436 +78.575539,1.451144,-0.767348,2.153182,0.959271 +76.916004,1.422748,1.236131,0.965397,0.926215 +22.456883,0.103438,-0.022279,0.277358,2.042076 +-2.033281,-0.160133,-0.751969,0.213197,0.671340 +-11.588289,-0.493123,0.813205,-1.256507,1.443632 +38.145425,1.007444,0.937916,0.184710,-1.254923 +-85.723658,-1.876553,-1.189667,-0.635362,0.619711 +-73.635926,-1.347126,-0.656894,1.200414,-0.971614 +-26.398962,-0.608227,-1.889649,0.311110,0.363116 +-59.083036,-1.237427,0.313184,1.328641,0.109408 +-78.135762,-1.496529,-1.449645,-0.083438,-0.650024 +-14.032448,-0.107030,-1.197878,-0.553649,-1.035242 +-43.599238,-1.116524,-0.548287,-0.155898,1.235812 +-53.086963,-1.402605,-0.692905,-1.243863,1.749577 +-22.336546,-0.387700,-1.255135,-0.477646,-0.413616 +-89.491338,-1.725807,-0.981166,1.194109,-0.677565 +-31.623210,-0.600639,1.852278,-0.601707,-0.291694 +1.665251,-0.248691,0.734878,0.490975,1.607346 +-25.193051,-0.622035,1.069771,0.496874,0.582738 +-47.173239,-1.081063,-0.309546,0.593101,0.615936 +22.197242,0.381935,0.238789,1.030283,0.430042 +-28.083514,-0.556119,-0.942558,1.669070,-0.130060 +177.006147,3.243093,-0.106337,-0.181449,2.307916 +-24.503489,-0.279993,-0.834282,0.515294,-1.279031 +28.713713,0.677926,-0.605715,2.157308,-0.487911 +13.235479,0.024219,0.452372,-0.079641,1.412221 +-2.683412,-0.297564,0.125576,-0.150056,1.375707 +-79.153255,-1.556582,0.850222,1.500760,-0.428115 +26.080959,0.478980,-0.510016,1.037540,0.333662 +5.325697,0.283751,1.249619,-0.846851,-0.987873 +-22.786342,-0.420187,-0.918652,-1.344451,-0.281785 +54.176545,1.126050,1.261922,-1.773032,-0.051394 +5.275723,0.059933,-1.308820,1.360659,0.277377 +19.097501,0.555513,-0.295090,-0.417367,-0.918687 +56.937147,1.252843,-1.968923,-0.692250,-0.448225 +2.871055,-0.182896,-0.799192,-0.645964,1.374876 +-26.978856,-0.706893,1.070611,1.649481,0.855556 +-68.215640,-1.366879,-1.801980,-1.170113,-0.224765 +-14.701331,-0.605861,-0.766657,1.992515,1.719378 +63.713567,1.188913,1.070150,0.351448,0.708304 +-48.849295,-0.811001,0.212740,0.258661,-1.114046 +82.012792,1.589839,-1.342128,1.399595,0.574071 +-43.344327,-0.839218,0.975545,0.331263,-0.309212 +-2.283269,-0.120948,-0.437458,-0.887492,0.419532 +110.306445,2.061504,0.592527,1.024063,1.208366 +98.449030,1.695723,1.023531,0.156694,1.897289 +-85.996163,-1.820603,-0.068634,-1.912255,0.270057 +-22.494967,-0.670115,0.487560,-1.176173,1.171718 +-42.638334,-0.452306,0.760415,-1.583903,-2.423879 +73.462933,1.562889,-0.299838,-0.243384,-0.273993 +56.721955,1.085896,0.817766,-0.025027,0.474698 +85.892434,1.809306,-0.652089,-0.847634,-0.218046 +-76.328698,-1.669405,0.570599,-0.662624,0.543360 +-77.968059,-1.199698,-0.828196,0.858014,-2.316144 +-23.567472,-0.457848,-1.348047,-0.370583,-0.159362 +79.706017,1.187386,0.325796,0.579633,2.589564 +40.908618,0.704507,1.410459,0.083827,0.789057 +42.978520,0.861636,-0.802824,-1.576996,0.139060 +42.319566,0.775155,-0.248529,0.234024,0.553040 +-23.913747,-0.454548,-0.080878,-0.091400,-0.218653 +-6.605917,0.208864,0.196861,-1.328186,-1.959670 +83.401261,1.476540,0.395804,-0.625563,1.380091 +-3.859246,0.169361,-0.903908,-0.111226,-1.413714 +31.077773,0.607897,0.194090,-0.446434,0.186609 +39.142576,0.815501,-0.356673,-0.183150,-0.048089 +33.896124,0.240753,-1.760763,0.565510,2.601683 +37.430839,0.951550,-0.267641,0.473472,-1.021162 +-9.687812,-0.013497,-1.220844,0.822545,-1.057711 +37.451106,0.913585,-0.271124,1.492689,-0.803179 +-23.714272,-0.434496,-0.478749,0.222134,-0.309172 +49.736484,1.149273,1.770801,-0.034988,-0.703176 +46.334124,1.049553,0.197600,1.317394,-0.535235 +-54.646016,-1.280304,-0.099176,0.650201,0.872457 +61.228274,0.926178,0.562969,-1.398568,1.909417 +84.957603,1.578118,-0.455540,0.869155,0.985450 +-36.789108,-0.717919,0.194108,1.179725,-0.230525 +-26.877469,-0.575638,-0.096060,2.560085,0.122010 +-9.578850,-0.453248,0.300474,0.326745,1.452468 +-18.338900,-0.245388,-0.815810,-0.889514,-0.753736 +93.872288,1.667224,1.781799,0.735622,1.523239 +-70.708031,-1.514470,0.018402,1.754933,0.321593 +-1.788763,-0.172802,-0.733156,0.441307,0.771920 +-24.890143,-0.571746,-0.222721,0.933128,0.332608 +-55.681358,-1.092313,0.141717,1.213098,-0.316408 +18.348642,0.384065,-0.089120,-2.067442,-0.032695 +20.904895,0.424067,-1.315816,-1.435910,0.039447 +-23.570130,-0.669073,1.826010,-0.605616,1.039905 +-6.789511,0.113270,-0.668144,0.919229,-1.438278 +23.025607,0.625667,0.482472,-1.070892,-0.857158 +-16.509658,-0.450065,-0.142379,-1.067620,0.622850 +38.116486,0.590655,0.507274,0.820482,1.108704 +15.526842,0.322082,0.823280,-0.813014,-0.011092 +-7.604632,-0.219672,-0.518270,1.477894,0.357113 +53.266213,1.255756,-0.439731,-0.186872,-0.894607 +-35.534891,-0.759133,1.876171,0.341756,0.150394 +68.114880,1.218195,-0.315087,0.872197,1.057368 +23.958108,0.060230,0.301547,-0.192361,2.463242 +-8.261245,-0.173072,-1.440051,-1.096275,0.015579 +-5.709861,-0.131257,-0.650003,-0.224856,0.076852 +-71.958045,-1.225766,1.047098,0.224452,-1.464375 +98.818956,2.013387,0.184680,-0.365322,0.136535 +49.064663,1.304340,1.126705,1.032546,-1.662492 +-51.933854,-1.037246,-1.382800,-0.875618,-0.190339 +-28.681693,-0.483561,-0.358029,0.491208,-0.612167 +111.745959,2.122156,-0.484234,-1.519370,1.032465 +-14.963619,-0.299677,-1.111967,2.620793,-0.050205 +-54.008273,-0.846794,0.856399,-0.446515,-1.514847 +3.966964,0.279969,0.129221,2.445752,-1.125489 +61.721780,1.066675,0.648710,1.382159,1.169296 +-4.620720,-0.098588,0.233216,0.708214,0.018850 +-70.349128,-1.153332,-1.355448,-1.793892,-1.687344 +1.898174,-0.223202,-0.828497,-1.600904,1.489863 +18.909322,0.517659,-0.755383,0.186767,-0.725744 +57.884234,0.515048,1.135566,0.570891,3.852731 +-7.367542,-0.234020,-0.075951,0.987335,0.466358 +-127.809679,-2.635748,0.924270,0.327821,0.003376 +-67.517171,-1.365824,1.796361,0.502784,-0.148969 +57.598113,1.103877,-0.589272,-0.002404,0.475168 +116.077897,2.403416,1.050654,0.201099,-0.057619 +32.961565,0.830336,-0.477657,0.071566,-0.856084 +2.406360,0.125225,0.543298,0.122298,-0.429406 +1.471337,-0.137449,1.314914,1.612278,0.952875 +-33.271948,-0.488849,-1.768439,0.140886,-1.119617 +24.444454,0.548884,0.327880,-0.125454,-0.254956 +49.593564,1.088770,-1.171654,0.018819,-0.376301 +-13.356460,-0.234153,0.767435,1.579213,-0.234137 +-44.416671,-0.782003,0.471542,-1.771069,-0.759703 +41.170334,0.950424,0.491919,-0.898415,-0.576904 +-5.254927,-0.230935,1.126565,1.848956,0.696206 +-144.195252,-2.906988,-1.631276,-1.038544,-0.374822 +19.705454,0.500240,-0.876774,1.220821,-0.533600 +-6.852548,-0.167118,-0.816936,1.206509,0.146714 +-20.864428,-0.471038,-1.407464,-1.448084,0.232050 +8.425794,0.164761,0.243953,0.173342,0.050888 +12.293810,0.499864,-2.100027,0.367649,-1.399306 +85.806603,1.450928,-1.024029,-1.682659,1.807197 +89.358875,1.687142,1.479944,-0.007973,0.881640 +-49.393801,-1.368315,0.105754,0.911363,1.987276 +46.981796,0.825416,0.021004,1.305479,0.813510 +85.879223,1.639965,-1.601966,0.075434,0.742127 +115.850708,2.319330,-0.309116,0.192049,0.393318 +-14.166986,-0.040158,-0.681052,0.128104,-1.430775 +36.135083,0.594754,0.281191,0.758929,0.853416 +85.854880,1.612221,-0.891192,-0.268531,0.896839 +35.734975,0.727630,-0.080717,0.732640,0.051946 +0.229338,-0.194908,-2.940389,0.593557,1.133770 +-28.979812,-0.730956,1.848609,0.685508,0.757922 +43.016920,0.951449,-0.924466,-1.703872,-0.366507 +-13.907509,-0.623769,0.217433,-0.190682,1.914031 +61.430945,1.181891,-0.528297,-0.202524,0.480908 +-42.942552,-0.927353,0.501094,0.975198,0.238369 +-9.828396,-0.201492,0.668340,0.598794,-0.006521 +-10.979241,-0.127655,2.052972,-2.929449,-0.560608 +86.784534,2.133033,0.588317,-0.151785,-1.952088 +-23.105487,-0.473839,0.006422,0.546284,-0.014452 +-22.910345,-0.358340,-0.181224,0.744192,-0.647542 +40.232111,0.857660,-1.002529,-0.019016,-0.159939 +63.271764,1.237654,-0.310308,-0.968046,0.379768 +15.946767,0.500917,0.751387,0.099332,-0.977555 +-29.943037,-0.264515,-1.239258,0.635418,-2.003862 +40.601153,0.455888,0.927840,-0.643518,2.165002 +35.944347,0.570613,-0.612237,-0.331308,0.968185 +38.279295,0.519347,0.401712,-0.108760,1.532739 +17.189412,0.408607,-0.886681,-0.838586,-0.307808 +39.704834,0.915390,-0.635558,-0.116766,-0.549540 +-8.449455,0.082440,-0.752156,-0.309209,-1.457551 +-43.911474,-0.783766,-0.831822,-0.916192,-0.690541 +-31.254562,-0.603981,-0.392471,-1.639484,-0.229549 +-20.688623,-0.288549,-0.856772,1.734937,-0.783754 +1.668326,0.087047,-1.987569,0.091761,-0.299007 +20.361102,0.513600,-2.872262,-1.169917,-0.532701 +-6.001539,0.242882,-0.548200,0.553149,-2.082099 +47.601730,0.619154,-0.728003,0.020794,2.057495 +-68.389571,-1.615132,-0.810252,-0.734592,1.164739 +60.785446,1.047318,-0.043477,-0.229391,1.169590 +-51.970801,-0.952316,0.830192,-0.772659,-0.676995 +30.600028,0.791032,-1.401851,1.402794,-0.909387 +10.552745,0.376410,1.125435,-0.869663,-0.902052 +11.954362,0.168655,1.410932,-1.090399,0.441941 +60.594607,0.964852,-0.592464,-1.234349,1.615583 +95.593864,1.964725,0.213980,-0.699726,0.035264 +16.927610,0.573744,0.292715,-1.288308,-1.276304 +-32.906163,-0.627734,-2.487809,1.418531,-0.288039 +28.078212,0.386809,1.023710,2.012270,1.090980 +30.952623,0.696954,0.369642,1.173125,-0.333819 +48.821877,1.105900,0.078143,-0.816217,-0.563947 +38.691960,0.654976,0.420192,-0.993863,0.810799 +-69.002925,-1.349576,0.111373,0.034027,-0.415214 +17.152741,0.714610,-0.336255,1.159074,-2.049941 +-40.739696,-0.783253,-1.230864,0.813517,-0.322062 +38.276981,0.710960,1.159330,-0.360966,0.444263 +-59.551267,-1.089633,-1.405567,-0.370508,-0.784762 +-97.878485,-1.519346,0.551741,-0.451159,-2.832156 +-25.790899,-0.564248,2.006093,1.542110,0.184551 +-30.117274,-0.607354,0.418206,0.425844,-0.077221 +79.328950,1.617213,-1.330314,-0.898784,0.104356 +-2.914316,0.175287,0.131928,-0.411823,-1.336725 +30.622220,0.576557,1.119575,3.078881,0.311250 +13.801043,-0.027515,-0.457096,1.661259,1.772252 +-19.725101,-0.467701,0.477041,-0.046921,0.346504 +85.102098,1.608781,-0.462940,1.090862,0.828230 +-1.948783,0.233786,0.833529,0.330880,-1.555896 +-126.669663,-2.238231,0.457687,-0.606865,-2.120700 +99.095721,2.303639,1.146441,1.259233,-1.479444 +108.766963,2.015275,-1.102270,-0.397195,1.290644 +-20.911083,-0.375196,0.557691,1.281644,-0.317715 +12.545889,0.573128,0.301107,-0.359630,-1.785866 +-31.857777,-0.624345,-2.205566,-0.604745,-0.184525 +-3.102783,0.202923,1.795878,1.547505,-1.515744 +39.098983,0.768207,3.926238,0.508269,0.215397 +51.680040,1.124353,-1.019664,0.564606,-0.334077 +42.990423,0.645376,0.686051,-0.964923,1.368632 +36.435469,0.534817,0.458387,-0.637308,1.228981 +-43.498551,-0.808494,0.328751,0.915402,-0.501757 +21.371322,0.372105,1.223569,-0.053121,0.389159 +-4.882158,0.032004,-0.677715,0.459972,-0.753418 +65.628407,0.963879,-1.369803,-0.557492,2.210523 +-45.466160,-0.926930,-1.024388,-3.241267,-0.059525 +-24.257059,-0.371858,1.653617,0.319525,-0.728462 +-0.011753,-0.020902,-0.591571,1.277665,0.117327 +-17.812900,-0.247177,-0.281100,-1.001620,-0.681984 +40.986422,0.695538,-0.071599,-0.293967,0.849102 +33.373605,0.949554,0.934320,-2.553921,-1.484898 +89.487862,1.991553,-0.147025,-0.550115,-0.832061 +42.524575,0.642723,0.709004,0.196521,1.329153 +38.099478,1.036088,-0.397558,-0.970124,-1.422975 +-14.240897,-0.259042,-0.037222,-0.071601,-0.196350 +-25.199474,-0.555547,-0.395681,-1.202150,0.204389 +11.504111,0.109395,0.223884,0.481009,0.725767 +31.779153,0.289775,-0.326024,0.871125,2.075401 +2.378890,-0.213443,-0.337086,0.148667,1.490726 +2.716467,0.048860,-0.662901,-0.701992,0.040592 +-18.578302,-0.464404,-0.251539,0.783391,0.462061 +47.837256,1.096469,-0.202981,-1.319247,-0.625677 +-43.511156,-0.787784,-0.472091,-0.168144,-0.620848 +96.318093,2.314659,-1.612716,0.686260,-1.867265 +-101.660329,-2.499406,-1.645399,-1.389572,2.290943 +14.520997,0.444774,0.364140,0.149363,-0.825654 +26.928391,0.319175,0.115026,-1.875172,1.340450 +-67.175480,-1.517174,-1.130069,-0.416194,0.750579 +-6.523258,0.052258,-1.284584,-0.717016,-1.060605 +6.866515,-0.006071,-0.098890,0.081829,0.838491 +114.425274,2.124771,-1.222895,1.920368,1.331339 +36.080741,0.413435,-1.244655,-0.773789,1.876796 +-19.136055,-0.246062,-0.175886,2.170943,-0.843247 +-36.849172,-0.702053,-1.463515,-0.392108,-0.327662 +96.656003,1.749584,0.689708,-1.292263,1.381454 +19.912258,0.181063,-0.412414,0.586166,1.303278 +20.550135,0.267050,1.065480,0.082284,0.889631 +33.993085,0.175211,-0.313530,0.367482,2.985259 +-48.815980,-0.881068,1.436335,1.186735,-0.712221 +18.950985,0.335058,-1.535572,0.469268,0.316156 +20.819823,0.425887,0.487872,-0.641487,0.019148 +-13.547434,-0.066080,0.047399,-0.651836,-1.211016 +-74.828752,-1.377669,0.513786,0.515035,-0.937825 +-89.612221,-1.979300,0.239247,-1.072743,0.747910 +-33.981594,-0.675178,-0.307962,-0.792420,-0.144519 +-14.487640,-0.256180,0.479442,-0.061764,-0.241497 +12.843026,0.371146,-0.155677,0.086590,-0.603985 +40.147546,0.615771,-0.450189,-0.139446,1.203884 +28.438875,0.330851,-1.077376,0.879417,1.451001 +33.044619,0.778361,-0.003374,-0.818199,-0.551186 +-7.231142,0.076822,-0.493757,0.996267,-1.282992 +22.902739,0.846771,-0.602821,-0.099094,-2.127227 +54.397143,0.897066,-1.137686,0.651522,1.274875 +-16.394150,-0.522723,-1.408461,-0.704344,1.049009 +-1.864271,0.097676,0.497998,0.024510,-0.773010 +-13.743958,0.064474,-0.144088,-0.939335,-1.975467 +-22.906456,-0.252568,-1.430141,1.632411,-1.247783 +-86.353794,-2.084113,0.287329,-0.287448,1.724697 +70.123403,1.100960,0.496699,-1.229550,1.958347 +56.681980,1.196572,-0.933268,-0.027305,-0.158530 +-4.234024,-0.073973,-1.385988,1.972542,-0.075666 +-18.136725,-0.469474,-0.465730,-0.463418,0.542560 +91.693106,1.901191,-1.513714,-0.708407,-0.060661 +42.551831,0.840734,-0.532169,-1.292625,0.207803 +-62.895569,-1.561493,-0.211667,-0.330120,1.503420 +-0.223739,-0.054894,0.645216,0.521122,0.285554 +-23.931246,-0.668090,-0.755745,-0.174960,0.992042 +12.520281,0.409763,0.697903,1.403370,-0.861088 +-48.443983,-1.013896,0.255384,-0.925425,0.085687 +-19.799870,-0.095464,-0.833056,-0.080600,-1.776246 +45.440854,0.840620,-0.366824,-0.238932,0.546734 +-34.220464,-1.130204,0.602118,1.516394,2.411677 +48.852498,0.665417,-0.378008,-0.878323,1.941217 +-93.505208,-1.993736,-1.209641,1.227669,0.374057 +-40.707571,-0.856498,-1.793106,-1.264498,0.097670 +-53.285959,-0.921860,0.069344,0.207267,-1.003957 +-14.787532,0.085893,-0.851406,-0.229800,-2.219300 +-61.815972,-1.515191,-0.249036,1.644968,1.366874 +25.991988,0.678947,-0.160360,-0.049029,-0.812402 +8.030046,0.260442,-1.962626,-1.008555,-0.538842 +-25.117915,-0.735530,0.609138,1.091310,1.236093 +18.348504,0.536510,-0.009119,0.028181,-0.898468 +24.635405,0.436739,-1.071054,1.235782,0.404295 +24.708270,0.500666,-0.660321,0.007063,0.049774 +-48.830849,-0.756351,-1.081548,-0.646573,-1.422254 +27.712862,0.555604,-0.151273,-0.197338,0.089581 +-34.087315,-0.607875,0.833334,-0.548269,-0.539123 +-90.022125,-1.893615,-0.817089,0.001205,0.213294 +-21.329391,-0.589365,-0.692910,0.357015,0.849602 +-22.272992,-0.319054,0.021312,1.076007,-0.796026 +-21.632396,-0.531214,0.681360,-1.102292,0.483877 +36.754854,0.860473,0.282580,-0.167122,-0.583077 +5.886278,0.450603,-0.283139,-1.161784,-1.869816 +-137.526774,-2.747505,1.388338,-0.526248,-0.499730 +33.575806,0.510792,0.854525,-0.657835,1.030581 +91.910281,1.855503,-1.611567,-0.346427,0.224235 +-4.001870,-0.142147,0.872315,1.029457,0.338687 +-54.219603,-1.111580,-1.567859,-0.531455,-0.035826 +4.873871,0.259723,-1.661520,0.638592,-0.904317 +114.941124,2.060748,0.971571,-0.248964,1.755341 +-25.661291,-0.496932,-0.035008,-0.989958,-0.182569 +-46.193889,-1.046911,0.718953,1.185704,0.536653 +20.810381,0.256030,1.014370,1.665474,0.982691 +-45.015817,-0.860413,-0.576892,1.006293,-0.384556 +22.910711,0.496714,1.523030,0.647689,-0.138264 +-33.032443,-0.612789,0.334457,0.285865,-0.387702 +-34.961496,-0.795557,-2.216819,1.279519,0.424394 +50.668787,0.969457,1.775311,-0.646227,0.427194 +-11.003652,-0.053905,2.320041,1.121858,-0.982304 +81.593908,1.738851,0.814152,1.832557,-0.321243 +-32.493980,-0.573662,-0.543425,-0.032753,-0.546859 +64.193621,1.221034,-0.959439,-0.226484,0.582098 +42.167659,0.883110,3.193108,-0.180480,-0.077837 +-27.201119,-0.718407,1.247742,-0.294950,0.894924 +-81.926073,-1.866540,0.790626,-0.684630,1.007514 +-11.309361,-0.158008,-1.654857,-1.012104,-0.426881 +35.624236,0.466671,1.747363,-0.948917,1.521006 +-13.331116,-0.335138,0.853976,1.554160,0.342338 +-43.482050,-1.021233,-0.564079,0.243801,0.708356 +62.833104,0.995582,-1.786375,-1.638029,1.703173 +-32.481385,-0.675708,-1.085825,-1.087246,0.034152 +-47.870125,-1.062394,0.985730,-0.187144,0.428307 +28.896078,0.379153,-2.390304,-0.596974,1.230222 +-15.612306,-0.262660,0.462484,0.960305,-0.336387 +1.681588,0.072037,0.077481,-0.951918,-0.212209 +-32.300419,-0.729602,0.572390,-1.293142,0.361417 +12.047486,0.055725,1.529550,-1.692465,1.094192 +-39.403919,-0.827590,-2.921350,-1.072139,0.086144 +-49.645544,-0.877983,0.367366,-0.226479,-0.826880 +-23.020813,-0.544919,1.103302,-0.037635,0.399136 +30.937179,0.610216,0.224216,-0.586531,0.156973 +14.194066,0.143049,0.582571,0.692858,0.849612 +33.475474,0.655901,-0.388518,-0.018709,0.194736 +-80.304555,-1.562546,-1.254289,0.794265,-0.529053 +-37.190233,-0.694713,0.152355,-0.524088,-0.409282 +27.851444,0.521942,0.346448,0.250493,0.296985 +5.970137,0.209659,-0.973546,0.807123,-0.491636 +54.064252,1.263707,0.199810,0.543479,-0.846316 +-14.248123,-0.245743,-0.054295,-2.696887,-0.272724 +-1.581479,-0.042823,-2.246889,0.848431,0.058023 +21.017288,0.249309,0.023903,0.144888,1.045088 +-59.163191,-1.006017,0.791663,1.158111,-1.214189 +-6.308441,-0.278883,-1.405190,-0.048897,0.845158 +-4.371930,-0.161286,0.174578,1.886186,0.404051 +-1.301508,-0.189120,1.511155,-0.127549,0.921650 +10.228018,0.120296,-1.124642,0.711615,0.514439 +-45.847015,-0.839722,-0.525755,-2.123896,-0.599393 +-50.707960,-0.716822,-0.121748,-0.082681,-1.866537 +-4.740566,-0.223463,-0.072829,0.473238,0.714000 +-45.357775,-1.091934,0.553251,0.171839,0.890248 +29.547676,0.811397,1.537932,0.818778,-1.148263 +5.915717,0.324057,-2.991136,0.085209,-1.147691 +5.174835,0.133541,0.956702,0.708109,-0.152470 +-44.991865,-0.828995,0.610370,0.747294,-0.560181 +37.898409,0.625508,0.123548,-0.592356,0.885231 +-17.421907,-0.629263,-1.707358,-0.535801,1.533728 +-26.878637,-0.391758,-0.322320,1.615376,-0.922410 +-19.893022,-0.348652,2.076748,-0.321635,-0.349258 +8.776043,-0.025554,-0.370614,0.543600,1.172729 +37.225571,0.698223,0.635172,0.895193,0.393485 +70.103057,1.594505,-2.153390,-0.991392,-0.846961 +-32.334315,-0.613403,0.170416,-0.388177,-0.302470 +-55.259857,-1.004141,0.234215,-0.034685,-0.767798 +-23.680551,-0.360039,0.312378,0.373349,-0.728077 +-80.319976,-1.371901,-0.209324,1.471170,-1.613561 +15.409722,0.230071,0.421587,0.665924,0.497743 +2.267065,0.073197,-0.083718,1.191707,-0.150243 +-49.916276,-0.785986,0.721563,0.303819,-1.381045 +12.871000,0.338496,2.270693,0.632782,-0.415288 +-27.608034,-0.676922,0.931280,1.031000,0.611676 +-15.877091,-0.194269,1.655407,1.048707,-0.755792 +51.914488,0.978422,-0.330789,-1.103670,0.522143 +47.207316,1.189470,0.701173,0.597400,-1.227608 +-2.785427,0.177701,0.610586,0.380198,-1.335344 +68.111688,1.712040,0.475431,0.434770,-1.747637 +47.634653,0.766080,-0.203674,-0.100154,1.226933 +24.493869,0.522251,0.889037,2.272435,-0.097915 +30.764412,0.871297,1.938929,0.126380,-1.345980 +-15.792703,-0.429244,0.462173,-1.598124,0.588553 +85.883753,1.804348,-1.293273,0.719758,-0.190904 +9.787300,0.096996,2.092387,-0.818221,0.595157 +52.017567,0.890198,-0.929511,-1.846188,1.035249 +0.410333,-0.134309,-1.748532,0.793489,0.810808 +11.528844,0.283288,-0.474904,0.586694,-0.258905 +-6.574331,0.040919,-0.513214,0.740824,-1.002187 +49.733365,0.869156,0.037938,-0.763286,0.887291 +-27.621629,-0.606700,-0.491902,1.200079,0.211284 +79.528706,1.464177,1.758620,0.280636,0.996859 +-97.818389,-1.831682,1.857702,1.497432,-1.051310 +0.442045,-0.087435,-0.059415,0.949595,0.548320 +50.991665,0.812526,1.003533,-0.072010,1.356240 +19.494786,0.496199,1.935154,0.511500,-0.535317 +84.701319,1.672572,-0.055769,-0.705012,0.419019 +3.594766,0.349650,-0.263448,0.095344,-1.564803 +38.933752,0.882333,0.265878,-0.470042,-0.452090 +-30.647922,-0.530501,-2.301921,-0.275052,-0.575818 +-45.331845,-0.769142,-0.193826,0.829475,-0.939903 +3.364084,-0.049464,0.382410,-1.122722,0.674819 +-89.400093,-1.975488,0.028458,-2.065083,0.751099 +-49.491035,-0.785989,0.507991,-1.836205,-1.331233 +27.421077,0.476358,2.759660,1.060210,0.505470 +41.101274,0.938284,-0.462275,0.096121,-0.516045 +50.610184,0.953125,0.516178,0.725096,0.513085 +-63.214225,-1.525656,2.558199,-0.551858,1.262584 +31.852687,0.722381,-0.399636,1.726964,-0.372833 +-17.588273,-0.070166,0.207688,0.429618,-1.660961 +-88.464104,-1.654887,0.093372,-0.122709,-0.960046 +5.260413,0.408253,0.472597,1.029156,-1.702584 +-6.938688,-0.269407,0.074095,1.502357,0.717542 +-96.661404,-2.018488,-0.122057,0.793176,0.145064 +62.456248,1.100821,0.361661,0.532006,1.061371 +-9.371075,-0.067178,1.487246,0.682052,-0.715760 +-73.494460,-1.778720,-0.055585,0.654366,1.496044 +-71.901603,-1.103367,2.493000,0.388579,-2.152891 +-42.354392,-0.956436,0.355613,1.484116,0.472406 +9.665231,0.191099,0.746254,-1.359856,0.046437 +44.867748,1.108183,-1.053416,0.612774,-1.039906 +21.183997,0.350630,1.109700,0.634721,0.489187 +-25.204894,-0.602212,0.301792,-0.998385,0.468774 +21.433070,0.428317,-0.165631,-0.593811,0.077156 +12.227700,0.197292,1.071543,1.699957,0.311309 +14.413645,0.323168,-1.594703,-0.466037,-0.147603 +90.460920,1.954157,1.481663,-1.058908,-0.505747 +29.656326,0.551476,-1.326472,0.390696,0.340589 +-50.240029,-0.985804,0.894647,1.194763,-0.284155 +-30.211410,-0.714681,0.168060,0.321679,0.521280 +29.442950,0.977816,1.386349,1.153590,-2.105647 +-34.312950,-0.790474,1.345420,1.882024,0.471468 +-55.730419,-1.179369,1.528031,0.671531,0.172254 +11.596208,0.308051,-0.343192,0.269127,-0.391649 +18.862222,0.224685,-1.760809,-1.418366,0.932591 +-0.132811,-0.045586,0.352055,-0.241236,0.243339 +34.288841,0.782874,0.550798,0.408053,-0.431126 +-35.717533,-0.650643,-0.863991,-0.592394,-0.487125 +30.046681,0.690144,0.012592,0.224092,-0.401220 +40.830804,0.715381,0.019617,0.438476,0.718186 +31.759406,0.678931,-0.340908,-0.519396,-0.136995 +-7.591012,0.071254,-0.918127,-0.695695,-1.293508 +-0.826011,-0.172627,-1.576392,0.652323,0.883660 +43.874916,0.872321,-0.808298,2.189803,0.183342 +13.101358,0.771699,-1.739714,1.148766,-2.848543 +115.176759,2.165056,1.026986,0.212574,1.190549 +16.906448,0.088407,-0.193659,-1.141689,1.477530 +-21.746485,-0.436720,-0.247026,2.099722,-0.066133 +-53.415953,-1.010598,0.664927,1.530195,-0.515218 +-48.434093,-1.143019,0.072874,-0.064689,0.820158 +-2.631360,-0.023552,-0.369527,0.225308,-0.174354 +-54.954697,-1.187598,0.413799,0.633777,0.309821 +-23.473225,-0.465310,-1.502970,2.644343,-0.105948 +10.940720,0.181866,-0.849844,-0.459361,0.248221 +32.263143,0.418398,-1.503080,0.650450,1.401599 +25.359949,0.721672,0.489375,-0.524520,-1.129052 +20.613311,0.179894,-1.570501,0.918317,1.392002 +-29.988207,-0.611769,-0.692421,-0.429302,-0.037037 +35.223990,0.638051,-0.542674,-1.801058,0.500844 +-2.764751,0.085318,0.522514,-0.830444,-0.808266 +-28.889546,-0.715304,0.216459,-0.730367,0.679598 +28.701139,0.491429,-0.098845,0.195482,0.569760 +19.284956,0.253165,0.454741,-0.033613,0.820344 +-0.255055,0.214094,0.385317,0.173181,-1.245739 +-11.664448,-0.034712,0.751933,1.142823,-1.168678 +-42.782831,-0.825497,-0.563725,0.412931,-0.321386 +24.808421,0.547097,1.098777,-0.217681,-0.202193 +-62.281194,-1.122545,0.271495,1.017661,-0.917503 +-14.930170,-0.626967,-0.562467,0.707752,1.812449 +33.335632,0.736844,0.515939,0.066991,-0.281328 +8.069530,0.298753,1.148446,-0.426358,-0.751791 +-4.743180,0.048522,-0.050238,0.270457,-0.830950 +-28.537607,-0.622649,-1.320023,-0.742471,0.194607 +-13.095268,-0.221254,0.815737,0.307407,-0.276813 +-24.921538,-0.300860,0.251474,0.388979,-1.209477 +2.652468,0.293558,-0.035641,0.466430,-1.356582 +-69.046695,-1.581191,0.146793,-0.483061,0.895038 +43.306319,1.073632,-0.700121,0.132970,-1.026515 +-39.964274,-0.989628,-0.224633,-0.982487,0.940771 +-19.340519,-0.731632,0.768840,0.049308,1.890441 +-39.505916,-0.772825,0.081874,-0.485364,-0.236819 +42.097830,0.919076,0.321698,0.267392,-0.290275 +-5.089397,-0.235970,0.266818,0.421546,0.744186 +-144.881640,-3.019512,1.238946,1.800511,0.183850 +35.418594,0.838941,-1.100154,-0.558302,-0.617253 +-7.335260,-0.112328,0.757508,0.614167,-0.220970 +-14.755751,-0.132634,-0.120381,1.107081,-0.974529 +-25.456229,-0.544383,0.375698,-1.150994,0.110923 +64.821941,1.276965,-1.075312,-1.207022,0.338023 +-44.464477,-1.050141,0.695203,1.370536,0.757495 +11.624904,-0.035826,0.821903,-2.619745,1.564644 +-56.422313,-1.090966,-0.214921,-1.105705,-0.410814 +-103.513051,-2.077812,0.360648,1.643378,-0.320298 +-61.073277,-1.200296,-0.653329,-0.474945,-0.334501 +-13.053166,-0.517288,-0.362839,2.298898,1.409347 +-99.305103,-1.831145,-0.086070,-2.075588,-1.228444 +13.044237,0.013002,2.720169,-0.264657,1.453534 +-46.103766,-0.823935,0.000683,2.049812,-0.719109 +13.514751,0.323079,-1.563191,-0.291811,-0.252354 +-53.548798,-1.209695,1.218762,1.530751,0.599929 +19.281787,0.259883,-1.320457,-1.236951,0.781823 +18.011152,0.235615,1.143754,-1.478586,0.770865 +0.469560,0.013929,-0.144360,0.198085,-0.024125 +-10.787184,-0.328618,-0.162793,-0.544114,0.603187 +69.436797,1.376496,1.466541,0.687066,0.313132 +-36.058708,-0.501784,-0.061004,0.322975,-1.372466 +-10.080897,-0.150320,-1.172234,-1.042578,-0.326696 +-2.106509,0.075805,-0.147057,0.975120,-0.677162 +41.746258,0.785800,-0.047711,-0.966976,0.425458 +46.924637,0.958386,1.074318,0.229075,0.051661 +19.423343,0.168461,1.139879,-1.006543,1.317598 +-0.826030,-0.077102,0.827183,0.276691,0.341152 +-8.281680,-0.308034,1.395684,1.310309,0.779661 +-20.127015,-0.379128,-1.014757,-0.581681,-0.203580 +-74.361561,-1.287164,1.038379,-0.583599,-1.397118 +-45.163913,-0.709580,-0.308483,-0.217017,-1.258505 +39.981734,0.659246,-0.762725,-1.607560,0.937570 +-43.504776,-0.985726,-0.792873,-0.530258,0.504047 +-6.374632,-0.134017,0.648280,-0.784898,0.014688 +-20.759861,-0.637387,-0.570746,1.420504,1.189017 +12.011867,-0.052404,-0.062004,1.246892,1.704106 +-71.199152,-1.731201,0.443002,0.041326,1.494938 +-56.686733,-1.193925,2.231300,1.732515,0.142943 +-50.043678,-1.193637,-0.670620,1.000582,0.919154 +-100.984519,-1.803140,0.508725,0.267127,-1.584136 +2.983288,0.020886,-0.616361,-1.260165,0.230701 +60.563058,1.275391,0.839469,0.533359,-0.151714 +-11.089998,-0.450013,0.358454,-0.535335,1.257149 +-26.635733,-0.487203,-1.296117,-0.769996,-0.351921 +30.426174,0.681953,-0.130143,0.324166,-0.310267 +-22.101602,-0.436386,-0.370011,-0.088363,-0.109610 +78.792358,1.513450,1.854093,-1.024187,0.630812 +-23.880608,-0.465806,-0.451659,-0.073948,-0.150838 +-0.997821,-0.094459,1.279465,-0.863883,0.419610 +22.142600,0.357787,1.053802,1.083051,0.560785 +-7.454662,0.014273,0.686318,-0.407036,-0.953939 +-119.255005,-2.650970,-2.073390,1.246085,1.091507 +-23.129794,-0.580523,0.394672,1.669905,0.588578 +-48.371726,-1.251539,1.117296,-0.082151,1.443765 +17.597788,0.347562,0.191577,0.355669,0.086698 +-10.067515,-0.003603,0.877362,1.503398,-1.158365 +1.111934,-0.026406,0.424061,-0.809604,0.280161 +57.111034,1.223083,-0.570351,0.352505,-0.258854 +-77.053386,-1.776012,1.384532,0.252569,1.063941 +2.268242,0.271579,1.053153,-1.081057,-1.276749 +-3.354548,0.045572,0.633919,2.143944,-0.651600 +33.512741,0.367287,-0.349317,-0.223466,1.838184 +-52.433682,-0.763259,0.048085,-1.627542,-1.804882 +-21.448359,-0.269875,0.377300,-0.444293,-0.978764 +4.519377,0.104201,-0.280675,-0.753965,-0.062593 +-68.918673,-1.406317,0.760056,-1.504720,-0.083106 +10.127135,0.105894,-1.564242,1.976441,0.584413 +-45.802438,-0.831955,-1.066235,-0.815376,-0.638283 +-38.900006,-1.110576,1.271555,0.935678,1.752270 +24.078347,0.414866,0.558140,0.043515,0.463289 +-57.660865,-1.119546,1.229215,0.178695,-0.393530 +-84.828151,-1.656623,0.721479,-0.735303,-0.524448 +-2.804653,-0.175564,1.295872,0.098068,0.668655 +-33.665356,-0.712846,1.503993,-0.254977,0.106430 +-14.578108,-0.327895,-0.867130,0.825098,0.155191 +-55.093470,-0.799941,1.001632,-0.393053,-1.908002 +56.502928,1.163164,0.462103,-0.981509,0.010233 +-76.823729,-1.351074,0.547265,0.481866,-1.322458 +38.265512,0.346710,-0.032281,-1.840078,2.511557 +85.418113,1.761188,0.449474,-1.181874,-0.000314 +95.220571,1.962587,1.341476,1.011463,0.003696 +11.855867,0.257550,-0.026514,-1.918771,-0.074446 +-9.195330,-0.445503,-0.522860,1.579572,1.453384 +-3.531133,-0.019420,-1.616311,0.799942,-0.303180 +-98.141143,-2.172670,-0.090533,-0.535328,0.847422 +1.791711,0.196131,0.257976,-1.548961,-0.904068 +-54.306016,-1.000331,0.179582,0.513908,-0.677745 +-75.423846,-1.466785,0.515628,0.975312,-0.501402 +-35.614170,-1.070852,0.189480,-1.280456,1.911419 +90.087932,2.056544,-0.926381,0.473632,-1.130888 +89.086373,1.765454,0.917862,-1.260884,0.404982 +14.686177,0.392416,-1.769076,-0.025574,-0.508964 +-25.583447,-0.443044,0.463185,0.296576,-0.479493 +-49.816583,-0.995148,0.178477,-2.177065,-0.181503 +-16.832850,-0.597510,0.978858,1.738900,1.422370 +16.746760,0.231446,-0.872961,-0.215668,0.646490 +-8.238905,-0.096624,0.608246,-0.945746,-0.415967 +72.026878,1.217159,-0.431620,0.998311,1.521316 +0.678036,-0.131473,-1.606577,-0.436764,0.826047 +-36.344711,-0.832356,0.632932,-0.552223,0.471416 +63.757595,1.105526,-1.143005,0.638730,1.187030 +-4.604388,0.241962,-0.562288,-1.724918,-1.913280 +19.537678,0.200569,0.061680,-1.015822,1.148637 +-37.127281,-0.550537,-0.147780,-0.508140,-1.220712 +48.152810,0.950016,-0.196281,-1.532701,0.243005 +-38.609401,-0.973379,-0.227476,0.336145,1.007133 +-28.774276,-0.359769,0.260281,-0.413465,-1.326048 +-17.707367,-0.709021,0.177750,-0.525880,1.953259 +-93.405665,-1.951102,0.759712,-1.817979,0.143588 +28.893813,0.436560,-1.009731,-2.362932,0.903935 +-3.692199,-0.351414,1.532368,-0.818429,1.563413 +1.813247,0.145521,1.588627,0.140983,-0.614123 +-12.124965,-0.170185,0.955305,0.696387,-0.453228 +-40.156480,-0.742484,1.685014,1.230875,-0.485306 +41.745142,1.009472,-0.091845,-1.499016,-0.844941 +-10.095493,-0.185288,-0.147002,0.043811,-0.129821 +16.841777,0.225264,1.702515,-1.269330,0.692723 +-82.451454,-1.594428,0.046981,0.005244,-0.599375 +3.901726,-0.039555,0.029756,0.028318,0.681501 +76.221557,1.551152,0.067518,1.179297,0.115675 +-25.149857,-0.504180,-0.487229,0.347676,-0.081523 +11.160592,0.113517,-1.237815,1.586017,0.662131 +-76.396711,-1.607661,0.006800,2.023606,0.184741 +0.323583,0.307802,0.743264,-1.348185,-1.710168 +-47.042409,-1.179040,0.201160,-0.464617,1.187679 +21.987348,0.603095,0.264775,2.485190,-0.850535 +46.256643,0.899600,0.629629,0.812862,0.307300 +-16.686025,-0.369277,0.408653,-1.776235,0.143388 +23.139777,0.679373,0.462591,0.666313,-1.148794 +-54.205438,-1.195883,-0.609783,1.196631,0.444603 +39.793632,0.753342,0.673181,1.289753,0.381158 +31.355082,0.683329,2.363872,1.072978,-0.209314 +-26.089039,-0.580053,0.472002,0.499685,0.239405 +-81.013612,-1.388543,0.159463,-1.217283,-1.600271 +92.744917,2.074083,-0.286004,-2.530288,-0.919385 +36.381039,0.559790,0.459180,0.833922,1.080781 +-48.392472,-1.320233,-0.469176,1.179440,1.831459 +-41.943950,-0.649278,-0.769973,0.034083,-1.223940 +82.141220,1.613711,0.964087,-0.244157,0.453534 +-21.799534,-0.176774,-0.284875,-0.491154,-1.548618 +49.220912,1.249505,-0.772569,-0.349439,-1.332777 +20.452186,0.622207,-0.075764,1.039112,-1.138833 +5.160095,0.000528,-2.296181,-1.443855,0.601207 +-12.192105,-0.412221,0.428186,0.537630,0.913474 +-17.792743,-0.322680,0.556230,1.328194,-0.250833 +20.458890,0.522835,2.142270,-0.024355,-0.573700 +-67.468406,-1.044809,-1.103208,2.056207,-1.966357 +92.613380,1.673452,0.829732,-1.299581,1.340461 +3.765963,-0.019638,1.364140,0.223914,0.552490 +-22.417147,-0.643550,-0.403648,-0.334775,1.029961 +-80.232568,-1.542802,-0.634286,0.596441,-0.632755 +30.360086,0.679143,0.731871,-0.329448,-0.302045 +7.814141,0.160574,1.190646,0.436938,0.003046 +-62.647985,-1.395789,-0.195390,-0.723990,0.591349 +-20.226385,-0.440044,-1.435862,1.441273,0.130741 +80.098095,1.362563,-1.123494,3.152057,1.640615 +-38.333188,-0.975873,2.632382,-0.949399,1.053642 +13.985381,0.326927,-2.211135,0.829406,-0.219101 +55.172745,0.975817,0.054934,-1.245717,0.918454 +16.132126,0.301700,-0.890318,-1.843794,0.175536 +-22.305862,-0.128538,0.092845,-0.548725,-1.881849 +26.990770,0.621572,-0.029263,0.377100,-0.369610 +-73.141828,-1.379319,1.794558,-0.033127,-0.730930 +-14.368907,0.078577,-0.979721,0.458168,-2.128734 +80.426368,1.846637,-0.691908,-1.525525,-1.070085 +-58.909457,-0.971657,0.862393,-0.626717,-1.379618 +118.026988,2.526932,1.044161,-0.489439,-0.530869 +8.463238,0.015618,1.519284,-0.907756,0.902277 +19.374998,0.403730,0.324359,-0.903702,-0.024196 +-72.240465,-1.415371,-0.802277,-0.342715,-0.420645 +1.213852,-0.230866,0.692919,-1.338606,1.453261 +37.280494,0.738467,-0.301104,-0.115648,0.171368 +123.353110,1.995667,-0.183197,0.606723,3.109919 +106.240231,1.904137,-0.159045,2.218780,1.625959 +43.192714,0.932192,-1.106294,1.135620,-0.236555 +15.588410,0.556553,-1.547304,0.486036,-1.335482 +-93.738805,-1.870792,1.676437,0.018418,-0.351513 +32.118610,0.640543,-1.299216,-0.113128,0.123078 +5.059099,0.057013,0.507836,1.528468,0.268592 +62.874109,1.317115,-0.607822,-2.121855,-0.118069 +94.540097,1.865775,0.656554,-1.191303,0.473833 +55.509165,1.041725,0.579857,-0.129517,0.583544 +17.820893,0.464370,-0.885249,0.316358,-0.550552 +-15.158440,-0.342549,-0.206611,-0.962733,0.170464 +13.822799,0.399688,0.586364,-0.528617,-0.651357 +-34.908483,-0.734174,1.455822,0.457280,0.081996 +-86.489070,-2.081929,-0.096713,0.211017,1.696456 +-17.171242,-0.138456,-0.850520,-0.209023,-1.224298 +-26.725651,-0.516430,-0.746578,-0.205859,-0.196467 +-19.793182,-0.342688,-0.777817,-1.407512,-0.371441 +-21.808750,-0.279810,-0.728272,0.505966,-0.964538 +57.903675,1.170199,0.554058,0.079598,0.134296 +-54.266640,-1.065114,-0.186971,-0.609512,-0.305225 +48.680969,0.953137,0.361504,-0.612437,0.287124 +7.893616,0.265362,-0.134279,-2.438817,-0.582759 +26.717704,0.428817,-0.367028,0.176442,0.693106 +0.010860,0.072681,-0.037571,-0.089234,-0.411493 +73.653356,1.393455,0.379640,0.429341,0.710549 +72.191436,1.390208,-1.311836,0.010353,0.557810 +-41.673025,-0.611518,-1.351685,-0.923233,-1.406661 +94.768012,2.075261,0.197911,1.735964,-0.689188 +28.839738,0.756989,1.355638,0.869606,-0.922165 +-24.826396,-0.479174,-1.196207,-1.106335,-0.185659 +71.859667,1.446978,-1.485560,1.031845,0.196555 +11.100262,0.225762,1.303508,-0.975366,0.017613 +69.434230,1.633432,-0.754276,0.302635,-1.146345 +94.072600,1.870965,0.534629,-0.868293,0.389614 +-17.095965,-0.399899,0.182501,-0.044893,0.269295 +16.301071,0.253740,0.097983,1.085786,0.467693 +21.488666,0.683569,0.261251,1.211944,-1.365956 +12.046548,0.160018,1.448499,1.117399,0.501783 +32.130618,0.552300,0.582027,-0.696772,0.625631 +36.315323,1.058424,-2.039232,-1.183259,-1.758739 +30.287068,0.693479,0.617006,1.059936,-0.392013 +-35.727477,-0.651418,0.424166,-0.320347,-0.483886 +92.664022,1.797687,0.572583,-0.571179,0.640843 +-20.573242,-0.456121,-0.557423,0.595029,0.181427 +4.827582,0.232787,1.194592,-2.189922,-0.756764 +10.026700,0.075955,-1.237662,0.482249,0.742682 +-3.363153,-0.018513,-0.827231,0.322719,-0.288659 +24.890289,0.399223,1.573987,-0.483186,0.647196 +-26.477548,-0.780199,0.893698,-1.196789,1.330575 +32.745331,0.541463,-2.591042,-0.576510,0.759155 +1.609208,0.343618,-0.385082,0.324084,-1.763040 +6.716008,0.170865,0.347582,0.018434,-0.183983 +6.411841,0.194384,-0.295401,0.338484,-0.353166 +-42.707499,-0.955123,-1.067533,2.062525,0.423599 +-46.440364,-1.012831,-1.412304,-0.908024,0.314247 +-57.549911,-1.304470,-0.939880,0.366598,0.669673 +15.291934,0.139887,0.157761,-1.390222,0.996118 +-19.624005,-0.382817,-1.184031,-0.397680,-0.123729 +-5.305926,-0.179093,0.734638,0.692856,0.395820 +-41.537671,-0.861703,0.876456,-2.152384,0.030031 +-12.269927,-0.190039,1.372848,-0.180392,-0.357445 +-70.949815,-1.154363,0.158053,-0.389924,-1.751829 +-27.343957,-0.487606,-0.420984,0.394452,-0.432558 +-45.842871,-1.092164,-1.545730,0.913772,0.834751 +-51.125283,-0.936506,-0.187329,0.292193,-0.667780 +76.643696,1.677701,1.628397,0.568983,-0.553588 +75.767908,1.399355,-0.646937,0.059630,0.924634 +46.433123,0.870068,0.364961,0.150419,0.495682 +-13.251291,0.078635,0.346488,0.916328,-1.998201 +23.983712,0.695402,-0.804966,-0.111509,-1.141009 +-45.187961,-0.824514,-1.056558,-0.528978,-0.608590 +66.675817,1.550500,-0.213989,0.984322,-0.998354 +-17.470983,0.007178,2.212531,0.160490,-2.086478 +50.250749,0.963376,1.896793,0.822060,0.412781 +32.173526,0.706079,1.230001,-1.026369,-0.242681 +-33.495466,-0.721738,-0.271655,-0.546680,0.176821 +37.466590,0.779349,-2.025808,-0.417910,-0.039020 +23.365530,0.670481,0.817890,-1.553759,-1.071857 +-2.032212,-0.335785,-1.503143,-0.259591,1.669022 +-22.491652,-0.268889,0.059218,2.573360,-1.106526 +49.108338,0.658544,-0.798297,-0.176947,2.010205 +-100.213687,-1.840874,0.026091,-0.624819,-1.279577 +38.978823,0.632408,-1.570225,0.621810,0.972554 +58.812362,1.167782,-0.411877,0.337603,0.254421 +-70.323806,-1.556629,1.754794,-1.280429,0.606010 +-6.507764,-0.104449,1.161878,0.070052,-0.168822 +53.405935,1.296063,-0.151299,-0.365506,-1.107157 +-63.584408,-1.372859,0.046935,-1.978225,0.351480 +17.749661,-0.088282,1.639117,-0.803675,2.579709 +19.786853,0.562881,-0.530971,1.987061,-0.879814 +67.989854,1.580541,-1.257578,-0.810858,-1.015094 +18.601461,0.224340,1.311984,-0.297482,0.904018 +-76.668745,-1.517874,0.575205,0.890383,-0.357029 +-36.668740,-0.718444,1.475356,0.310908,-0.213447 +12.554325,0.317453,-0.529332,-0.093387,-0.332860 +-19.313369,-0.549557,-0.045411,-0.387131,0.859588 +-31.555644,-0.482744,1.624678,0.122670,-0.953329 +-24.432405,-0.509807,-0.658248,-2.488327,0.034441 +-38.242766,-0.581782,2.067906,-0.197938,-1.173880 +-60.224853,-1.053682,1.710613,0.950308,-1.067803 +47.524552,0.900399,-1.178041,1.183494,0.451225 +-54.918400,-0.963759,-0.048652,0.343788,-0.957151 +14.522086,0.368673,1.278452,0.028745,-0.393339 +25.567486,0.249384,0.279022,-0.095296,1.577453 +-4.583559,0.281009,-0.158154,1.012637,-2.132596 +-125.221331,-2.529560,-1.151014,0.241322,-0.296607 +-19.579552,-0.228600,-0.191028,-2.562334,-0.994349 +21.021767,0.613518,-1.668584,-0.257377,-1.022793 +-35.466411,-0.637740,-0.555477,-0.623141,-0.530997 +-42.290382,-0.638962,1.009817,1.642015,-1.323090 +-28.661548,-0.451949,-0.215610,-0.007045,-0.789339 +-110.508705,-2.151815,-0.987180,-0.211130,-0.719153 +42.497611,1.051948,0.250200,-0.383971,-0.998061 +105.389371,2.281652,-1.880010,-1.535040,-0.617642 +-1.380709,-0.107302,-1.126767,-1.570876,0.447717 +-29.056470,-0.443282,1.711708,-0.172946,-0.884803 +17.215947,0.320786,0.286956,-1.270061,0.194052 +44.953433,1.195047,0.377212,-0.558922,-1.523187 +75.569458,1.614779,-1.406815,1.325432,-0.322028 +36.914028,0.506241,-1.049655,0.568103,1.447306 +-22.989535,-0.186896,0.777407,1.205995,-1.630473 +-25.522945,-0.606503,-0.694600,-0.459090,0.455904 +20.543404,0.400481,-1.195294,-0.077443,0.131065 +16.591857,0.296120,-0.234587,0.005113,0.261055 +-57.255754,-1.317132,-0.752435,-1.002217,0.776028 +23.671733,0.264020,0.288724,0.732492,1.272354 +-14.217121,-0.319848,2.089539,-0.835143,0.151758 +56.712634,1.428858,0.329107,-0.585928,-1.474132 +6.823869,0.114228,-0.056946,-0.363612,0.150302 +67.204299,1.628616,-0.055548,-1.703382,-1.380101 +37.489672,0.698828,-0.526035,0.492019,0.420973 +-0.303095,-0.064138,0.421921,0.321357,0.328762 +18.408561,0.550052,-1.334025,0.105376,-0.968344 +-11.155907,-0.238110,1.172757,-0.904614,0.046003 +1.472200,-0.064078,-0.162124,-2.196128,0.536291 +41.846845,0.996048,1.501334,-1.421811,-0.756795 +54.259315,1.200262,1.153901,-0.411427,-0.463161 +73.233162,1.676393,1.137315,-1.153351,-0.945507 +6.604370,0.091752,0.214713,-0.116229,0.252240 +-38.915377,-0.892788,-1.968504,0.034714,0.513614 +12.030480,0.361636,1.538037,0.361396,-0.645120 +10.685507,0.123205,1.695051,0.043602,0.551485 +12.614775,0.185176,1.278866,0.222290,0.425445 +49.543400,0.887504,0.388312,0.060086,0.760847 +-30.998905,-0.680025,-0.714351,0.293072,0.232254 +-8.754455,-0.372207,1.543244,1.884586,1.088749 +-13.589576,-0.471932,-1.077745,0.064280,1.088951 +6.690391,-0.222475,-0.403076,-0.674934,2.046862 +4.528719,0.199060,-0.385314,0.069802,-0.600217 +-10.487547,-0.220964,-2.041735,0.208383,0.026886 +47.107226,1.192735,0.590422,0.731380,-1.257868 +1.142709,0.216776,-0.837262,-0.588867,-1.097303 diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 818c863df0da..91dcb827292e 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -747,6 +747,7 @@ def test_ranking_prediction_early_stopping(): with pytest.raises(AssertionError): np.testing.assert_allclose(ret_early, ret_early_more_strict) + def test_ranking_with_position_information(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' params = { @@ -771,7 +772,9 @@ def test_ranking_with_position_information(tmp_path): position = np.random.randint(28) positions.append(position) out_file.write(f"pos_{position}\n") + positions = np.array(positions) + # test setting positions through Dataset constructor lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params, position=positions) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) @@ -785,6 +788,17 @@ def test_ranking_with_position_information(tmp_path): gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] + # test setting positions through set_position + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + lgb_train.set_position(positions) + gbm_unbiased_set_position = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_set_position.best_score['valid_0']['ndcg@3'] + + # test get_position works + positions_from_get = lgb_train.get_position() + np.testing.assert_array_equal(positions_from_get, positions) + # add extra row to position file with open(str(tmp_path / 'rank.train.position'), 'a') as file: file.write('pos_1000') @@ -794,6 +808,7 @@ def test_ranking_with_position_information(tmp_path): with pytest.raises(lgb.basic.LightGBMError, match="Positions size \(3006\) doesn't match data size"): lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) params = { From d92f6d05180c54dab226ab6e3890e745d182204b Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 08:09:02 +0000 Subject: [PATCH 79/97] remove position_filename --- docs/Advanced-Topics.rst | 3 ++- docs/Parameters.rst | 4 ---- include/LightGBM/config.h | 4 ---- include/LightGBM/dataset.h | 5 +---- src/io/config_auto.cpp | 6 ------ src/io/dataset_loader.cpp | 4 ++-- src/io/metadata.cpp | 17 +++++++---------- 7 files changed, 12 insertions(+), 31 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 58a8afcf72bd..ebb1dd919044 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -94,7 +94,8 @@ LightGBM can make use of positional data, for example provided in an additional It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consisting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). -The position file corresponds with training data file line by line, and has one position per line. The position file path can be specified using the parameter ``position_filename``, or through the ``Dataset`` constructor when using Python API. +The position file corresponds with training data file line by line, and has one position per line. And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. +In this case, LightGBM will load the position file automatically if it exists. The positions can also be specified through the ``Dataset`` constructor when using Python API. If the positions are specified in both approaches, the ``.position`` file will be ignored. Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM `_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item. During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels. diff --git a/docs/Parameters.rst b/docs/Parameters.rst index db2ae180ed4b..0e85091d51cb 100644 --- a/docs/Parameters.rst +++ b/docs/Parameters.rst @@ -1141,10 +1141,6 @@ Objective Parameters - used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. -- ``position_filename`` :raw-html:`🔗︎`, default = ``""``, type = str - - - path to the file storing positions used in lambdarank - Metric Parameters ----------------- diff --git a/include/LightGBM/config.h b/include/LightGBM/config.h index a70dd270b12c..b96843b3fcc7 100644 --- a/include/LightGBM/config.h +++ b/include/LightGBM/config.h @@ -969,10 +969,6 @@ struct Config { // desc = used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. double lambdarank_position_bias_regularizer = 0.0; - // type = str - // desc = path to the file storing positions used in lambdarank - std::string position_filename = ""; - #ifndef __NVCC__ #pragma endregion diff --git a/include/LightGBM/dataset.h b/include/LightGBM/dataset.h index e1c49f2539da..e7baa42dc2e6 100644 --- a/include/LightGBM/dataset.h +++ b/include/LightGBM/dataset.h @@ -52,9 +52,8 @@ class Metadata { /*! * \brief Initialization will load query level information, since it is need for sampling data * \param data_filename Filename of data - * \param position_filename Filename of positions */ - void Init(const char* data_filename, const char* position_filename); + void Init(const char* data_filename); /*! * \brief init as subset * \param metadata Filename of data @@ -342,8 +341,6 @@ class Metadata { void InsertQueries(const data_size_t* queries, data_size_t start_index, data_size_t len); /*! \brief Filename of current data */ std::string data_filename_; - /*! \brief Filename of ranking positions */ - std::string position_filename_; /*! \brief Number of data */ data_size_t num_data_; /*! \brief Number of weights, used to check correct weight file */ diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp index 0925596fe5a3..746ddafa46b9 100644 --- a/src/io/config_auto.cpp +++ b/src/io/config_auto.cpp @@ -305,7 +305,6 @@ const std::unordered_set& Config::parameter_set() { "lambdarank_norm", "label_gain", "lambdarank_position_bias_regularizer", - "position_filename", "metric", "metric_freq", "is_provide_training_metric", @@ -624,8 +623,6 @@ void Config::GetMembersFromString(const std::unordered_map>& Config::paramet {"lambdarank_norm", {}}, {"label_gain", {}}, {"lambdarank_position_bias_regularizer", {}}, - {"position_filename", {}}, {"metric", {"metrics", "metric_types"}}, {"metric_freq", {"output_freq"}}, {"is_provide_training_metric", {"training_metric", "is_training_metric", "train_metric"}}, @@ -1047,7 +1042,6 @@ const std::unordered_map& Config::ParameterTypes() { {"lambdarank_norm", "bool"}, {"label_gain", "vector"}, {"lambdarank_position_bias_regularizer", "double"}, - {"position_filename", "string"}, {"metric", "vector"}, {"metric_freq", "int"}, {"is_provide_training_metric", "bool"}, diff --git a/src/io/dataset_loader.cpp b/src/io/dataset_loader.cpp index 4dfc6e1dab24..354936cfb01a 100644 --- a/src/io/dataset_loader.cpp +++ b/src/io/dataset_loader.cpp @@ -225,7 +225,7 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac } dataset->data_filename_ = filename; dataset->label_idx_ = label_idx_; - dataset->metadata_.Init(filename, config_.position_filename.c_str()); + dataset->metadata_.Init(filename); if (!config_.two_round) { // read data to memory auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, rank, num_machines, &num_global_data, &used_data_indices); @@ -312,7 +312,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename, } dataset->data_filename_ = filename; dataset->label_idx_ = label_idx_; - dataset->metadata_.Init(filename, config_.position_filename.c_str()); + dataset->metadata_.Init(filename); if (!config_.two_round) { // read data in memory auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, 0, 1, &num_global_data, &used_data_indices); diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index dad64125bbf8..d0cac7c027f6 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -21,15 +21,13 @@ Metadata::Metadata() { position_load_from_file_ = false; query_load_from_file_ = false; init_score_load_from_file_ = false; - position_filename_ = ""; #ifdef USE_CUDA cuda_metadata_ = nullptr; #endif // USE_CUDA } -void Metadata::Init(const char* data_filename, const char* position_filename) { +void Metadata::Init(const char* data_filename) { data_filename_ = data_filename; - position_filename_ = position_filename; // for lambdarank, it needs query data for partition data in distributed learning LoadQueryBoundaries(); LoadWeights(); @@ -554,7 +552,7 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) for (data_size_t i = 0; i < num_positions_; ++i) { - positions_[i] = map_id2pos.at(positions[i]); + positions_[i] = positions[i];//map_id2pos.at(positions[i]); } } @@ -598,14 +596,14 @@ void Metadata::LoadWeights() { } void Metadata::LoadPositions() { - if (position_filename_ == std::string("")) { - return; - } num_positions_ = 0; - TextReader reader(position_filename_.c_str(), false); + std::string position_filename(data_filename_); + // default position file name + position_filename.append(".position"); + TextReader reader(position_filename.c_str(), false); reader.ReadAllLines(); if (reader.Lines().empty()) { - Log::Fatal("Position file '%s' is not found.", position_filename_.c_str()); + return; } Log::Info("Loading positions..."); num_positions_ = static_cast(reader.Lines().size()); @@ -623,7 +621,6 @@ void Metadata::LoadPositions() { position_load_from_file_ = true; } - void Metadata::LoadInitialScore(const std::string& data_filename) { num_init_score_ = 0; std::string init_score_filename(data_filename); From b55e44b0612a32818e246a1cd48d2ae6e0322e04 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 08:24:16 +0000 Subject: [PATCH 80/97] remove useless changes --- src/io/metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index d0cac7c027f6..78414f445ee0 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -552,7 +552,7 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { #pragma omp parallel for schedule(static, 512) if (num_positions_ >= 1024) for (data_size_t i = 0; i < num_positions_; ++i) { - positions_[i] = positions[i];//map_id2pos.at(positions[i]); + positions_[i] = map_id2pos.at(positions[i]); } } From fdda50f8894a974f6d0657c9e864e3993ebd4adb Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Tue, 8 Aug 2023 16:31:32 +0800 Subject: [PATCH 81/97] Update python-package/lightgbm/basic.py Co-authored-by: James Lamb --- python-package/lightgbm/basic.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index 85b0c7d881c6..fd096e92f304 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -2197,10 +2197,6 @@ def construct(self) -> "Dataset": group_info = np.array(self.reference.group).astype(np.int32, copy=False) _, self.group = np.unique(np.repeat(range(len(group_info)), repeats=group_info)[self.used_indices], return_counts=True) - if self.reference.position is not None: - position_info = np.array(self.reference.position).astype(np.int32, copy=False) - _, self.position = np.unique(np.repeat(range(len(position_info)), repeats=position_info)[self.used_indices], - return_counts=True) self._handle = ctypes.c_void_p() params_str = _param_dict_to_str(self.params) _safe_call(_LIB.LGBM_DatasetGetSubset( From 56d77c24a7eb66253dde7aacf03ac9f55722562f Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 08:35:25 +0000 Subject: [PATCH 82/97] remove useless files --- tests/c_api_test/preb.txt | 500 ------------ tests/python_package_test/data.csv | 100 --- tests/python_package_test/data_dask.csv | 1000 ----------------------- 3 files changed, 1600 deletions(-) delete mode 100644 tests/c_api_test/preb.txt delete mode 100644 tests/python_package_test/data.csv delete mode 100644 tests/python_package_test/data_dask.csv diff --git a/tests/c_api_test/preb.txt b/tests/c_api_test/preb.txt deleted file mode 100644 index f35c133463e6..000000000000 --- a/tests/c_api_test/preb.txt +++ /dev/null @@ -1,500 +0,0 @@ -0.72816922538206474 -0.34254413558637198 -0.39375878909343448 -0.53665466907657056 -0.38072398950936409 -0.34017648118647142 -0.45978203210567342 -0.44628065737977557 -0.60080289251541041 -0.53744413394458634 -0.42064207683507732 -0.60885892984431689 -0.56631892989602972 -0.59384597288021679 -0.41509342657650794 -0.6447246230140683 -0.41398638532950588 -0.49619015828026219 -0.50816544640733008 -0.5433913020081389 -0.70765788333666124 -0.58737495344973312 -0.55202925139748704 -0.49384895389338523 -0.52715450986408574 -0.62003028099221735 -0.5271304222108627 -0.61143126453405039 -0.51040226321315518 -0.53516911727194239 -0.58265400049973981 -0.60839584467348884 -0.32045717477916147 -0.50436461383302911 -0.56626759697004803 -0.42529703085324583 -0.39216853111023542 -0.31062337068447893 -0.39695081862302423 -0.56257576053762914 -0.21787838987724456 -0.51317474641799277 -0.43137842267424936 -0.5632093944281632 -0.41152383935988024 -0.18558569537594216 -0.56585961941753482 -0.48065327283132342 -0.74050687924153469 -0.37556583740692273 -0.82519659752152996 -0.5302330114162398 -0.22432282431040096 -0.66954005712518405 -0.62112099036121549 -0.52787120676714061 -0.32934423532118201 -0.37602098166850595 -0.43298291586822835 -0.52925150429120693 -0.46250078955240304 -0.60757346403046508 -0.74149872758318369 -0.44285821827108729 -0.67928949090859736 -0.43382497854499585 -0.51231584748043701 -0.247140869791797 -0.7226842943309939 -0.57036909154325477 -0.41340599994509702 -0.5685276873486278 -0.71280318144675847 -0.38721005503175732 -0.71966468278946993 -0.76854678984409386 -0.48834863757820024 -0.40267567760863782 -0.70338647717389102 -0.61775482428378359 -0.41428705286435008 -0.56361320875176002 -0.6442020838419894 -0.70651580109729517 -0.20778472573516621 -0.41478764955120984 -0.48078639190418138 -0.59362686094208328 -0.44758202367995031 -0.64385479183149297 -0.35720388319190521 -0.35754670393997789 -0.49232989003411853 -0.56636920108340971 -0.48569817022544198 -0.63277119005777882 -0.43393064976357837 -0.43556665306809156 -0.7331161120590588 -0.27973037887813385 -0.45416195308273949 -0.43089514054677458 -0.54906154859737255 -0.36168897508855047 -0.47064286495658053 -0.67113198006622732 -0.5362421344673165 -0.57324619386279518 -0.34414679089845102 -0.52348226159323219 -0.41676568695820876 -0.49108911497066665 -0.44330930549031294 -0.24876192766094601 -0.58714982657373627 -0.71013870537805623 -0.34397499263868941 -0.57155232362819841 -0.41048575310259899 -0.49737430255200454 -0.47189096802485031 -0.66764025079844513 -0.52923772808164837 -0.46340444214827337 -0.35608724781996315 -0.46420421801911049 -0.51668476760268423 -0.64831275301692892 -0.43918091422652927 -0.48277245420321302 -0.55849430746574891 -0.47611787420285684 -0.48562808680585734 -0.43264795894625369 -0.38066444735962551 -0.67075756760210925 -0.41051782077733057 -0.67536163223484957 -0.46664307460700077 -0.52459413309980529 -0.4746240504700428 -0.58405071264679709 -0.52499392446764903 -0.50962484356825899 -0.55937379802764442 -0.35217297281347715 -0.38315020415159362 -0.48029668437134981 -0.67027252713953178 -0.42114818018884226 -0.33896134332400352 -0.66532937236872891 -0.50027400566434232 -0.41849942912247773 -0.53283022597370455 -0.37819833075377918 -0.42174948103710375 -0.47016465206864677 -0.48970903509423114 -0.42453311555017942 -0.73636490994064541 -0.52010828356038052 -0.64082625083353251 -0.22300684912301558 -0.54468162436264855 -0.45938588802016334 -0.60439824927919816 -0.64378129238701565 -0.36465235254606332 -0.68695643355221081 -0.74540016978378254 -0.51080984365545212 -0.43131123272188487 -0.6896752324217279 -0.47581100181976643 -0.66577981888510618 -0.49983121317630602 -0.40372208440824964 -0.35920836556409236 -0.18236290456819801 -0.24263318676107182 -0.48323045259130437 -0.24094872612509108 -0.48035420002415574 -0.37747678847086152 -0.73672532220921483 -0.50478964632958079 -0.46308123007068225 -0.72593723350992023 -0.50683500485100796 -0.53762469363299326 -0.4176841526900435 -0.55185622044794536 -0.47546001640382751 -0.5522531876403215 -0.59346320905690964 -0.45915342031396184 -0.46100132693087015 -0.30310709558752225 -0.52849719589140509 -0.43300081514979633 -0.27302253150363232 -0.46146687612990839 -0.49419972338446916 -0.53541256633700318 -0.28231826885884193 -0.54800509322342883 -0.46092780871857986 -0.65946375879983188 -0.30311257538621356 -0.32448644717174041 -0.60602844828214997 -0.5266144627430438 -0.42260572901779836 -0.41973578749864188 -0.43773105150144687 -0.65203538865860078 -0.5190971064815445 -0.46870737786818895 -0.37819632742074116 -0.64222288958825147 -0.29285772401646853 -0.5782993574365668 -0.50816383026400025 -0.40459310252704089 -0.72451394301377325 -0.3988911605759109 -0.38948876687559641 -0.58786635655785735 -0.4567731124678342 -0.52254990688142922 -0.342216292004832 -0.41025189598037087 -0.60922959301919655 -0.49801830351262411 -0.45509521650260859 -0.61839273426823838 -0.51948512079115539 -0.51957016625028241 -0.54859674358076826 -0.49775636069388524 -0.28559977239767326 -0.22559120432624344 -0.51533796680029975 -0.43457110449375641 -0.32310354449152678 -0.3470205901517695 -0.66162630391137311 -0.41159419987099966 -0.35347673353871789 -0.37267078033390405 -0.43309743695371639 -0.38197279548033053 -0.67686196129957121 -0.67492684647667178 -0.56623301854706309 -0.44958559907571466 -0.66507670329237445 -0.46754769110307898 -0.46277629753232408 -0.46278185023237489 -0.49261676851939318 -0.32840852227458994 -0.55009590938324393 -0.55740327930252231 -0.60208788647637956 -0.59694564464334576 -0.37164726510937918 -0.55942019678447774 -0.19967172616498283 -0.32230119630976839 -0.48031834660795408 -0.37105438680830116 -0.49448349233626626 -0.69982993678588823 -0.61657392311280412 -0.52721190658957773 -0.71731539988068571 -0.37359486367193034 -0.41351991255222226 -0.43421480490214148 -0.51422498200331535 -0.30111836867200181 -0.41756836465088154 -0.48635495811950902 -0.3983806388716683 -0.46676700527201159 -0.42954972869201297 -0.38207465958363551 -0.54058439311800122 -0.53482361057077665 -0.44111059747635689 -0.34302590111088133 -0.66230380938398514 -0.28452505834512065 -0.40382652376986966 -0.40473071866671301 -0.40948918067824103 -0.41710643416750176 -0.32732661890008058 -0.39110854415326002 -0.42963519644513043 -0.48061784046160183 -0.61922246525791003 -0.35928763954401505 -0.66598022986198879 -0.39155974102201996 -0.72525893075604475 -0.53880138638651998 -0.5336261241365855 -0.65057732717760763 -0.50749372436514861 -0.43631033184402251 -0.73423630368236259 -0.49201633161964392 -0.61920314297939205 -0.57285661477171756 -0.51855006294612416 -0.42934983013423655 -0.40543433340051949 -0.53198765341485688 -0.47542585769879675 -0.52745815798800433 -0.57114368037229968 -0.59534514535110339 -0.44005221197410377 -0.44939601527291062 -0.44826114468524919 -0.4323617741571773 -0.458566154799254 -0.47314581804815653 -0.58744189351626297 -0.63675203666532387 -0.5804116726857933 -0.24756701756211835 -0.41454919412031699 -0.50434233050585342 -0.44163240481900445 -0.7015347144176427 -0.37889569458155642 -0.58278502159514001 -0.34447989922720518 -0.53332757736827097 -0.4233394870228937 -0.71945766050826132 -0.34937745895084515 -0.51547576377624804 -0.6729458724709223 -0.43721697013092459 -0.51293251676173734 -0.47611565769859765 -0.66858319468077898 -0.56444042514485959 -0.3232993461576506 -0.5336272685400133 -0.38312680818919731 -0.46833368528524411 -0.42722984600517683 -0.76493746129395712 -0.42153961662322126 -0.41681766033430345 -0.43929719874856737 -0.69500378552300468 -0.38928035790199378 -0.43147025074078554 -0.55920452412531474 -0.42894793416255833 -0.43442162935635986 -0.503212059848682 -0.68012930263242488 -0.48914064710841854 -0.27612129532323215 -0.53431245370499159 -0.6045973005562707 -0.63649103198737655 -0.50529097096682396 -0.58855254626783615 -0.60332172849418841 -0.60948433205877073 -0.21033368146558756 -0.40105860535592164 -0.76576952226902295 -0.56032901397782431 -0.49716119079344812 -0.54671301465650146 -0.50339186720895035 -0.57222958060106743 -0.53092125118898159 -0.47977652816344601 -0.68804390053570019 -0.67587717113717383 -0.45151198485657995 -0.58903092289861969 -0.41270368434201915 -0.72868809400599199 -0.48976693927324211 -0.46315791511471843 -0.69203149057457658 -0.69066004578091489 -0.22752843011573498 -0.44746859510224962 -0.56300744465883035 -0.57267158573149934 -0.53039800754269206 -0.53381382719534376 -0.66539039062918004 -0.56105480141003827 -0.59681767781266593 -0.46630916540536593 -0.39569253482998978 -0.64214418716534494 -0.41401841078073093 -0.41803364939949533 -0.66694748698812323 -0.45533672480245041 -0.47959617759024908 -0.78622928107673162 -0.59044680460965016 -0.46270650723991447 -0.52966423011216834 -0.59309508150314827 -0.45487655762459989 -0.44987884334688288 -0.69559467323198632 -0.37758323022318574 -0.46928051850278013 -0.48270761970812393 -0.59196315446282577 -0.40657323467155099 -0.4060445606251889 -0.34813812777382963 -0.67773822398445416 -0.54567747328268879 -0.71457208044119724 -0.52993603546746437 -0.50745178395133106 -0.43442266648832961 -0.46072147155862792 -0.53823902515603284 -0.7552831359895138 -0.46753380199833788 -0.44249910937574538 -0.43451356469444918 -0.48287308211005392 -0.6834407324164935 -0.5871760108194507 -0.46838801465223917 -0.69859760734015108 -0.46225488812609877 -0.70602904909814235 -0.25980202402639374 -0.5151635985827917 -0.5079515548240946 -0.49757674154255982 -0.45555838090674267 -0.42074825662008747 -0.79027206514030135 -0.47253419563245308 -0.3793242141395845 -0.50481275780964652 -0.43369027374583646 -0.42404092819642814 -0.39147593198624142 -0.48342051831462446 -0.32219711323686961 -0.5942662336530895 -0.62387595564165399 -0.45508434052140312 -0.45190156205406901 -0.44650683617222847 -0.32343032516157727 -0.44318145059549174 -0.54742356980393136 -0.56828808304948708 -0.56799254528844034 -0.59471453737286029 -0.46580382092460726 -0.55244497014089167 -0.54880384388723835 -0.493056760399538 -0.56765364950637343 -0.55994378859402938 -0.55320906145280446 -0.58838693902959793 -0.55917435186993925 -0.6504168677181329 -0.43055030187935561 -0.5076695055093613 -0.60108248996680347 -0.55195483098733944 -0.47746471327886564 -0.44391354440749742 -0.48833752671540848 -0.36657132142256094 -0.68656415852126007 -0.71930502856513934 -0.39245119176277071 -0.550361588643836 -0.41242219282318993 -0.49308951662419243 diff --git a/tests/python_package_test/data.csv b/tests/python_package_test/data.csv deleted file mode 100644 index db7d1d702d9c..000000000000 --- a/tests/python_package_test/data.csv +++ /dev/null @@ -1,100 +0,0 @@ -8.648289,-1.918771,-0.026514,-0.074446,0.257550 -22.812042,1.586017,-1.237815,0.662131,0.113517 -116.259508,-1.519370,-0.484234,1.032465,2.122156 --41.296264,0.813517,-1.230864,-0.322062,-0.783253 --44.592294,1.632411,-1.430141,-1.247783,-0.252568 --74.928338,1.158111,0.791663,-1.214189,-1.006017 -41.292330,-2.619745,0.821903,1.564644,-0.035826 -84.306971,-1.260884,0.917862,0.404982,1.765454 --32.890778,-0.601707,1.852278,-0.291694,-0.600639 --27.378431,0.341756,1.876171,0.150394,-0.759133 --40.078349,-3.241267,-1.024388,-0.059525,-0.926930 --19.549355,-1.150994,0.375698,0.110923,-0.544383 --29.478456,0.822545,-1.220844,-1.057711,-0.013497 --14.679364,1.441273,-1.435862,0.130741,-0.440044 -19.421082,0.005113,-0.234587,0.261055,0.296120 --24.953130,-1.106335,-1.196207,-0.185659,-0.479174 -51.248443,0.822060,1.896793,0.412781,0.963376 -44.964201,0.686260,-1.612716,-1.867265,2.314659 --33.948808,0.324084,-0.385082,-1.763040,0.343618 --47.256284,0.915402,0.328751,-0.501757,-0.808494 --4.692470,1.179440,-0.469176,1.831459,-1.320233 --25.178353,0.173181,0.385317,-1.245739,0.214094 -45.718943,0.812862,0.629629,0.307300,0.899600 --4.564190,0.091761,-1.987569,-0.299007,0.087047 --34.048333,-0.114540,1.237816,1.353872,-1.713135 -7.950252,1.402794,-1.401851,-0.909387,0.791032 --82.789212,0.515035,0.513786,-0.937825,-1.377669 -45.173689,-1.607483,0.184634,1.307143,0.227460 --44.914698,-1.328186,0.196861,-1.959670,0.208864 --15.369559,0.975120,-0.147057,-0.677162,0.075805 -69.845248,-0.192361,0.301547,2.463242,0.060230 -56.481809,1.305479,0.021004,0.813510,0.825416 --11.091510,-0.730367,0.216459,0.679598,-0.715304 --18.912247,1.158596,-0.820682,0.787085,-0.974682 -90.350893,-1.191303,0.656554,0.473833,1.865775 --38.081046,-0.392108,-1.463515,-0.327662,-0.702053 --9.631647,-1.550663,0.068563,-0.503476,0.099651 --30.786857,-0.889514,-0.815810,-0.753736,-0.245388 -4.356590,1.886186,0.174578,0.404051,-0.161286 --8.660092,0.322719,-0.827231,-0.288659,-0.018513 --78.909071,-0.661786,0.852433,0.186454,-2.025143 --2.636663,0.361396,1.538037,-0.645120,0.361636 -19.805827,0.324166,-0.130143,-0.310267,0.681953 -2.518608,-1.070892,0.482472,-0.857158,0.625667 --8.152970,0.069802,-0.385314,-0.600217,0.199060 -29.770925,0.250493,0.346448,0.296985,0.521942 --27.444725,0.244967,-0.506943,0.243687,-0.822220 -35.318364,-0.115648,-0.301104,0.171368,0.738467 --16.114609,1.579213,0.767435,-0.234137,-0.234153 --11.356382,1.031000,0.931280,0.611676,-0.676922 -0.651292,1.477894,-0.518270,0.357113,-0.219672 -31.204561,-0.019016,-1.002529,-0.159939,0.857660 --43.266465,0.331263,0.975545,-0.309212,-0.839218 -63.449672,-0.108760,0.401712,1.532739,0.519347 -10.251606,0.473238,-0.072829,0.714000,-0.223463 --76.543649,-0.446515,0.856399,-1.514847,-0.846794 --43.030150,0.412931,-0.563725,-0.321386,-0.825497 -10.196025,0.064280,-1.077745,1.088951,-0.471932 --15.924806,2.143944,0.633919,-0.651600,0.045572 --36.012281,0.504987,0.865755,-0.114736,-0.792521 --33.392396,1.142823,0.751933,-1.168678,-0.034712 -86.423410,2.153182,-0.767348,0.959271,1.451144 -6.129007,0.276691,0.827183,0.341152,-0.077102 --51.221118,-2.123896,-0.525755,-0.599393,-0.839722 --49.704076,0.747294,0.610370,-0.560181,-0.828995 --81.013025,-0.460639,1.057122,-0.719844,-1.478522 --32.461177,0.058209,-1.142970,0.153725,-0.883857 --17.082839,0.024510,0.497998,-0.773010,0.097676 -57.383351,-0.315269,0.758969,0.651391,0.954002 --13.195178,-1.448084,-1.407464,0.232050,-0.471038 --70.213079,-0.342715,-0.802277,-0.420645,-1.415371 --21.858771,0.293072,-0.714351,0.232254,-0.680025 -35.110923,-0.151785,0.588317,-1.952088,2.133033 --31.118078,-0.919424,1.549934,0.473592,-1.062304 -84.232583,-0.990536,-0.566298,2.190456,0.586857 -33.205762,0.443819,0.774634,-0.707669,1.266911 --58.936136,-0.474945,-0.653329,-0.334501,-1.200296 --42.273368,-1.724918,-0.562288,-1.913280,0.241962 -2.340775,1.277665,-0.591571,0.117327,-0.020902 -40.279993,-0.264657,2.720169,1.453534,0.013002 -30.173963,1.083051,1.053802,0.560785,0.357787 -16.824428,0.647689,1.523030,-0.138264,0.496714 --5.368955,-0.208122,-0.493001,-0.622700,0.280992 -17.166576,-0.217681,1.098777,-0.202193,0.547097 -70.785717,-0.072010,1.003533,1.356240,0.812526 -43.069118,-0.012247,-0.897254,0.628346,0.624120 --35.638264,0.310908,1.475356,-0.213447,-0.718444 --1.219277,0.357015,-0.692910,0.849602,-0.589365 -32.156035,-1.236951,-1.320457,0.781823,0.259883 -23.652480,-0.898415,0.491919,-0.576904,0.950424 --7.942392,0.097078,0.968645,0.513267,-0.529760 -126.701586,0.570891,1.135566,3.852731,0.515048 --4.640738,-0.463418,-0.465730,0.542560,-0.469474 --33.422466,-0.908024,-1.412304,0.314247,-1.012831 -41.198261,2.189803,-0.808298,0.183342,0.872321 -48.530080,-0.981509,0.462103,0.010233,1.163164 -20.295596,-0.818221,2.092387,0.595157,0.096996 -17.658867,0.224092,0.012592,-0.401220,0.690144 -54.625086,0.067528,-1.424748,-0.225776,1.465649 --38.533085,-0.485364,0.081874,-0.236819,-0.772825 diff --git a/tests/python_package_test/data_dask.csv b/tests/python_package_test/data_dask.csv deleted file mode 100644 index 83e2eb959c32..000000000000 --- a/tests/python_package_test/data_dask.csv +++ /dev/null @@ -1,1000 +0,0 @@ -56.738133,1.239584,0.604121,1.068379,-0.396230 --6.159212,-0.164335,-1.015879,-0.993359,0.212093 -25.505362,0.493318,0.700310,-0.858358,0.184836 --15.630020,-0.476221,0.193590,0.262561,0.874389 --21.310863,-0.529760,0.968645,0.097078,0.513267 --51.704342,-1.104863,-0.513196,1.217959,0.220541 --82.951301,-1.692957,-1.103589,-0.988591,-0.098340 --8.817806,-0.026521,-0.744903,-0.163067,-0.881875 --77.858664,-1.478522,1.057122,-0.460639,-0.719844 --64.662621,-1.175595,-0.948348,0.597228,-0.895070 --29.052979,-0.562168,-0.805870,-1.683438,-0.209222 --63.495550,-1.534114,-0.748487,0.332314,1.277677 -40.551412,0.823171,-1.295079,-1.289961,0.073318 --6.366368,-0.185846,1.218971,-0.056588,0.309999 --65.837724,-1.097368,1.191899,-0.739371,-1.476925 -5.133149,0.326133,-0.184902,0.924027,-1.251114 -1.896450,0.257753,-0.155259,0.334176,-1.241761 --36.887882,-0.302181,-0.064184,-0.361311,-2.603137 --20.211424,-0.649373,-0.600424,1.419603,1.321304 -27.517167,0.664703,-0.497571,1.001825,-0.552921 --13.287606,-0.339122,0.457806,-0.123733,0.370061 --106.217377,-2.087027,-0.909683,-0.031059,-0.584618 --15.598486,-0.103431,1.594442,2.095844,-1.239053 --1.023335,0.159856,-0.866175,1.265708,-1.027675 -20.523607,0.342725,0.447709,0.569767,0.456753 --4.887067,0.032797,-0.924233,-0.230401,-0.758495 --3.055117,-0.039307,1.431367,0.333860,-0.134497 -37.239465,0.505589,-0.404397,2.271450,1.489113 -17.744286,0.189582,0.677875,-2.703232,1.001046 --47.479018,-1.062304,1.549934,-0.919424,0.473592 -72.906480,1.593187,-0.125787,-0.989605,-0.511216 --13.025066,-0.110665,-0.369207,0.841984,-0.896642 --23.193862,-0.517611,1.188393,-0.016423,0.223788 --37.381345,-0.727137,0.620672,-0.074433,-0.247519 --3.603126,0.077368,0.538910,1.523124,-0.861284 -121.397723,2.426716,-0.564774,-1.377618,0.432960 -100.082249,1.873298,1.281016,-0.447322,1.080048 -2.983104,0.195025,0.622711,-1.130615,-0.758282 -110.804925,2.184097,-0.611385,-0.580648,0.570507 --69.869577,-1.531108,-0.062191,0.572057,0.514255 -62.613652,1.124113,0.407052,-0.772878,0.947526 --30.282812,-0.673491,2.145149,-0.835347,0.278994 -7.946695,-0.089736,1.800940,-0.676392,1.440117 -30.950050,0.444198,1.346226,0.466545,1.101323 --43.659691,-1.189412,0.638392,-0.900621,1.642673 -51.905761,1.180641,0.051198,0.045223,-0.627313 -26.187586,0.494030,-0.671623,-0.550305,0.260674 --32.882334,-0.964705,1.590582,-0.284385,1.628469 -13.606900,0.453509,0.446873,0.058984,-0.982294 --39.418565,-0.792521,0.865755,0.504987,-0.114736 -41.260883,1.081767,1.328933,0.622070,-1.312219 --4.181337,-0.140375,1.352203,-0.524567,0.307613 -67.374861,1.021383,-1.879924,-1.585983,2.088443 -48.908565,0.881408,1.355443,-0.916274,0.721135 -41.936281,0.756789,-0.244080,-1.016683,0.612469 -24.979850,0.432263,-0.712174,-0.707626,0.470044 -30.884532,0.835692,1.441569,0.529804,-1.129707 --71.527626,-1.713135,1.237816,-0.114540,1.353872 -65.401599,1.392465,0.260322,0.288694,-0.250047 --96.630682,-2.025143,0.852433,-0.661786,0.186454 --23.147713,-0.546244,0.183360,-1.478912,0.391804 -35.637195,0.624120,-0.897254,-0.012247,0.628346 --4.760304,0.228996,0.298158,0.603248,-1.857901 -62.711268,1.296995,-0.504775,-0.999302,-0.022868 --54.545646,-1.143726,-0.208117,-0.033230,0.108560 --26.436378,-0.601368,0.440475,-1.592994,0.319782 -17.816529,0.409819,1.899882,0.672574,-0.241258 -47.170779,0.586857,-0.566298,-0.990536,2.190456 -4.204770,-0.015310,-0.973069,0.119580,0.579291 -39.632693,0.858451,-1.448591,0.038238,-0.234621 -0.533436,0.099651,0.068563,-1.550663,-0.503476 --34.513746,-0.658116,-0.819258,-1.345871,-0.303726 -59.426661,1.200532,-1.117186,-1.967070,0.140360 --76.786718,-2.135674,0.223239,1.056057,3.137749 --33.969474,-0.513867,0.955142,-0.062679,-1.059214 -35.267866,0.538296,-0.839210,-0.364953,1.072507 --7.417766,-0.021367,0.884045,-2.424240,-0.747212 --53.186174,-1.222128,-0.374821,-0.240325,0.712998 --41.555817,-0.883857,-1.142970,0.058209,0.153725 --40.551873,-0.974682,-0.820682,1.158596,0.787085 -22.125167,0.585904,0.218534,-1.415267,-0.736770 -31.915628,0.183835,-1.004055,0.349800,2.693034 --34.971223,-0.889241,1.472671,0.876047,0.955301 --99.880047,-1.907808,1.887688,-0.413606,-0.860385 -77.754511,1.683928,-0.038508,1.078681,-0.458884 -1.385733,0.295624,-1.167780,1.476934,-1.516643 --46.911212,-0.895346,0.651136,-0.995815,-0.408101 -48.488916,1.082691,1.325797,-0.093636,-0.471125 -7.271360,0.056650,0.486502,-0.070499,0.529693 -86.540739,2.024310,-0.661982,0.189706,-1.363174 --51.802485,-1.111458,1.140149,0.498222,0.246505 --94.183293,-2.211862,-0.266652,-1.423957,1.533434 -51.833880,0.954002,0.758969,-0.315269,0.651391 --14.364838,-0.127918,0.203464,-1.606446,-0.955540 -44.351450,0.996571,-0.190503,0.640480,-0.466495 -92.510681,2.086047,0.416446,-0.361463,-1.014759 -18.485142,0.712712,0.437313,-0.372319,-1.883150 --94.003921,-1.869742,0.449219,0.190424,-0.388521 -5.985189,0.088658,-0.316073,-0.617652,0.197316 --15.919441,-0.278963,0.340051,0.790372,-0.279760 -54.775949,1.201214,-1.008086,-2.038125,-0.408075 -8.410255,0.338553,-1.025753,0.216625,-0.937926 -37.689252,0.672861,-0.573602,-0.354041,0.591814 -5.762199,0.022466,1.114322,-1.180813,0.547119 -49.781995,0.645484,0.219150,-0.307778,2.163255 -38.549010,0.742095,1.561511,1.301741,0.299293 --19.511945,-0.149941,0.493655,-0.524662,-1.433152 --35.857591,-0.614323,-1.371743,0.978890,-0.709789 --47.358007,-0.654076,1.373659,0.511203,-1.830633 --49.513008,-0.905732,1.374438,-0.595661,-0.653766 -12.278915,0.166452,2.455300,0.289169,0.492451 -17.728568,0.331980,-0.435486,0.709452,0.190500 --36.095843,-0.935439,0.808058,-0.535963,1.085982 --27.141358,-0.362441,1.160827,-1.294681,-1.119670 --22.144920,-0.502975,-0.234408,0.294224,0.263487 --32.826274,-0.539760,-0.978373,0.195845,-0.778305 -1.100857,-0.258796,-0.295480,0.560919,1.598647 -60.240622,1.238283,1.702215,0.308833,0.021272 --58.487004,-0.924564,-0.213457,-0.327017,-1.597599 -48.844404,0.681891,-0.359292,0.583928,1.846707 -5.439839,-0.005596,-0.387100,-1.091701,0.668742 -27.968954,0.439501,1.674492,0.457773,0.778937 -55.403758,1.266911,0.774634,0.443819,-0.707669 -64.068543,1.065172,-0.836921,-0.916321,1.452617 -83.949823,1.897767,0.764030,-0.834323,-0.947893 -18.169315,0.101856,-1.467525,-1.239107,1.549020 -31.180192,0.782130,1.081985,-0.779853,-0.790899 --15.189929,-0.313058,0.604515,-1.250408,-0.000709 --89.009748,-1.448014,-0.502054,0.440014,-2.198806 --15.403408,-0.385022,-2.130566,0.141257,0.382989 --0.480558,0.435975,0.126178,0.682069,-2.532246 -27.728969,0.558327,-0.920674,0.538756,0.076005 --126.684931,-2.471645,-0.203045,0.577072,-0.796895 -16.991130,0.386323,1.573020,1.755476,-0.204471 -70.432730,1.022570,0.563909,1.384273,2.439752 -33.505332,0.534506,1.795211,-0.320668,0.887655 --5.830533,-0.045512,0.329509,-0.569833,-0.424236 -40.096885,0.850898,-0.103222,0.390465,-0.137372 -69.158404,1.465649,-1.424748,0.067528,-0.225776 -42.172756,0.662881,-1.296832,0.181022,1.173474 -43.176080,0.680216,0.319652,-1.778588,1.192508 -10.790567,0.463206,-1.232523,0.848174,-1.367138 --95.070988,-2.153343,-0.862776,-0.478837,1.097153 --19.340279,-0.238948,0.755391,-0.576771,-0.907564 --61.478584,-1.029372,1.112688,-1.522359,-1.352670 -86.675134,1.923446,-0.471264,-1.689183,-0.774615 --15.163206,0.025388,-0.689728,-0.013838,-1.919673 -87.515288,1.727543,0.120031,0.038003,0.436324 --28.502093,-0.718832,-0.064188,-0.194697,0.745005 -123.716946,2.412615,-0.262891,-0.019260,0.784604 --19.041372,-0.103255,1.661470,-0.175854,-1.643189 -30.861879,0.425490,-0.709441,-1.371674,1.197247 --27.945422,-0.412998,-0.848429,-0.031439,-0.926698 --27.424262,-0.641482,0.754291,0.800410,0.431923 -26.530196,0.482067,-1.927673,0.393797,0.368733 -75.369204,1.565524,1.881157,-0.555200,-0.065750 -17.285247,0.190636,0.397927,-1.032524,0.941311 --87.930527,-1.970104,0.730764,-1.211172,0.892597 -10.574060,0.067856,-0.846357,0.484733,0.852774 --30.314904,-0.822420,-0.009300,0.000207,1.121031 --39.345829,-0.800039,-0.457302,1.237438,-0.063525 -8.310568,0.280992,-0.493001,-0.208122,-0.622700 -25.103271,0.012499,1.058729,-1.668599,2.868403 --70.487875,-1.451176,0.363632,-1.252393,-0.012089 -35.199118,0.840644,-1.889541,-0.446183,-0.652624 -23.749863,0.202329,1.818062,-0.733033,1.631857 -23.670387,0.998010,-0.139590,2.088375,-2.896255 -40.012818,0.638660,0.148089,0.367620,1.058118 -22.669947,0.513106,0.615367,0.738810,-0.259547 -39.422200,0.886887,0.198948,-2.604214,-0.420762 -36.866459,0.874517,-1.042044,-1.203201,-0.649765 -48.831332,0.921802,0.606851,0.420094,0.482688 --37.798006,-0.822220,-0.506943,0.244967,0.243687 -23.664217,0.629530,-0.631578,0.895355,-0.804316 -10.419259,0.257199,-0.325611,0.007880,-0.240665 -58.935380,0.892954,0.712322,-0.409687,1.829620 -3.090739,0.502745,1.629166,-0.965665,-2.493271 -22.195611,0.227460,0.184634,-1.607483,1.307143 --59.681008,-1.085151,1.244680,2.949094,-0.825411 --0.444153,-0.139397,1.318302,-1.909356,0.739653 --6.553380,-0.092944,0.186416,1.116834,-0.239502 --14.140045,-0.688150,-0.324831,0.981765,2.252436 -78.575539,1.451144,-0.767348,2.153182,0.959271 -76.916004,1.422748,1.236131,0.965397,0.926215 -22.456883,0.103438,-0.022279,0.277358,2.042076 --2.033281,-0.160133,-0.751969,0.213197,0.671340 --11.588289,-0.493123,0.813205,-1.256507,1.443632 -38.145425,1.007444,0.937916,0.184710,-1.254923 --85.723658,-1.876553,-1.189667,-0.635362,0.619711 --73.635926,-1.347126,-0.656894,1.200414,-0.971614 --26.398962,-0.608227,-1.889649,0.311110,0.363116 --59.083036,-1.237427,0.313184,1.328641,0.109408 --78.135762,-1.496529,-1.449645,-0.083438,-0.650024 --14.032448,-0.107030,-1.197878,-0.553649,-1.035242 --43.599238,-1.116524,-0.548287,-0.155898,1.235812 --53.086963,-1.402605,-0.692905,-1.243863,1.749577 --22.336546,-0.387700,-1.255135,-0.477646,-0.413616 --89.491338,-1.725807,-0.981166,1.194109,-0.677565 --31.623210,-0.600639,1.852278,-0.601707,-0.291694 -1.665251,-0.248691,0.734878,0.490975,1.607346 --25.193051,-0.622035,1.069771,0.496874,0.582738 --47.173239,-1.081063,-0.309546,0.593101,0.615936 -22.197242,0.381935,0.238789,1.030283,0.430042 --28.083514,-0.556119,-0.942558,1.669070,-0.130060 -177.006147,3.243093,-0.106337,-0.181449,2.307916 --24.503489,-0.279993,-0.834282,0.515294,-1.279031 -28.713713,0.677926,-0.605715,2.157308,-0.487911 -13.235479,0.024219,0.452372,-0.079641,1.412221 --2.683412,-0.297564,0.125576,-0.150056,1.375707 --79.153255,-1.556582,0.850222,1.500760,-0.428115 -26.080959,0.478980,-0.510016,1.037540,0.333662 -5.325697,0.283751,1.249619,-0.846851,-0.987873 --22.786342,-0.420187,-0.918652,-1.344451,-0.281785 -54.176545,1.126050,1.261922,-1.773032,-0.051394 -5.275723,0.059933,-1.308820,1.360659,0.277377 -19.097501,0.555513,-0.295090,-0.417367,-0.918687 -56.937147,1.252843,-1.968923,-0.692250,-0.448225 -2.871055,-0.182896,-0.799192,-0.645964,1.374876 --26.978856,-0.706893,1.070611,1.649481,0.855556 --68.215640,-1.366879,-1.801980,-1.170113,-0.224765 --14.701331,-0.605861,-0.766657,1.992515,1.719378 -63.713567,1.188913,1.070150,0.351448,0.708304 --48.849295,-0.811001,0.212740,0.258661,-1.114046 -82.012792,1.589839,-1.342128,1.399595,0.574071 --43.344327,-0.839218,0.975545,0.331263,-0.309212 --2.283269,-0.120948,-0.437458,-0.887492,0.419532 -110.306445,2.061504,0.592527,1.024063,1.208366 -98.449030,1.695723,1.023531,0.156694,1.897289 --85.996163,-1.820603,-0.068634,-1.912255,0.270057 --22.494967,-0.670115,0.487560,-1.176173,1.171718 --42.638334,-0.452306,0.760415,-1.583903,-2.423879 -73.462933,1.562889,-0.299838,-0.243384,-0.273993 -56.721955,1.085896,0.817766,-0.025027,0.474698 -85.892434,1.809306,-0.652089,-0.847634,-0.218046 --76.328698,-1.669405,0.570599,-0.662624,0.543360 --77.968059,-1.199698,-0.828196,0.858014,-2.316144 --23.567472,-0.457848,-1.348047,-0.370583,-0.159362 -79.706017,1.187386,0.325796,0.579633,2.589564 -40.908618,0.704507,1.410459,0.083827,0.789057 -42.978520,0.861636,-0.802824,-1.576996,0.139060 -42.319566,0.775155,-0.248529,0.234024,0.553040 --23.913747,-0.454548,-0.080878,-0.091400,-0.218653 --6.605917,0.208864,0.196861,-1.328186,-1.959670 -83.401261,1.476540,0.395804,-0.625563,1.380091 --3.859246,0.169361,-0.903908,-0.111226,-1.413714 -31.077773,0.607897,0.194090,-0.446434,0.186609 -39.142576,0.815501,-0.356673,-0.183150,-0.048089 -33.896124,0.240753,-1.760763,0.565510,2.601683 -37.430839,0.951550,-0.267641,0.473472,-1.021162 --9.687812,-0.013497,-1.220844,0.822545,-1.057711 -37.451106,0.913585,-0.271124,1.492689,-0.803179 --23.714272,-0.434496,-0.478749,0.222134,-0.309172 -49.736484,1.149273,1.770801,-0.034988,-0.703176 -46.334124,1.049553,0.197600,1.317394,-0.535235 --54.646016,-1.280304,-0.099176,0.650201,0.872457 -61.228274,0.926178,0.562969,-1.398568,1.909417 -84.957603,1.578118,-0.455540,0.869155,0.985450 --36.789108,-0.717919,0.194108,1.179725,-0.230525 --26.877469,-0.575638,-0.096060,2.560085,0.122010 --9.578850,-0.453248,0.300474,0.326745,1.452468 --18.338900,-0.245388,-0.815810,-0.889514,-0.753736 -93.872288,1.667224,1.781799,0.735622,1.523239 --70.708031,-1.514470,0.018402,1.754933,0.321593 --1.788763,-0.172802,-0.733156,0.441307,0.771920 --24.890143,-0.571746,-0.222721,0.933128,0.332608 --55.681358,-1.092313,0.141717,1.213098,-0.316408 -18.348642,0.384065,-0.089120,-2.067442,-0.032695 -20.904895,0.424067,-1.315816,-1.435910,0.039447 --23.570130,-0.669073,1.826010,-0.605616,1.039905 --6.789511,0.113270,-0.668144,0.919229,-1.438278 -23.025607,0.625667,0.482472,-1.070892,-0.857158 --16.509658,-0.450065,-0.142379,-1.067620,0.622850 -38.116486,0.590655,0.507274,0.820482,1.108704 -15.526842,0.322082,0.823280,-0.813014,-0.011092 --7.604632,-0.219672,-0.518270,1.477894,0.357113 -53.266213,1.255756,-0.439731,-0.186872,-0.894607 --35.534891,-0.759133,1.876171,0.341756,0.150394 -68.114880,1.218195,-0.315087,0.872197,1.057368 -23.958108,0.060230,0.301547,-0.192361,2.463242 --8.261245,-0.173072,-1.440051,-1.096275,0.015579 --5.709861,-0.131257,-0.650003,-0.224856,0.076852 --71.958045,-1.225766,1.047098,0.224452,-1.464375 -98.818956,2.013387,0.184680,-0.365322,0.136535 -49.064663,1.304340,1.126705,1.032546,-1.662492 --51.933854,-1.037246,-1.382800,-0.875618,-0.190339 --28.681693,-0.483561,-0.358029,0.491208,-0.612167 -111.745959,2.122156,-0.484234,-1.519370,1.032465 --14.963619,-0.299677,-1.111967,2.620793,-0.050205 --54.008273,-0.846794,0.856399,-0.446515,-1.514847 -3.966964,0.279969,0.129221,2.445752,-1.125489 -61.721780,1.066675,0.648710,1.382159,1.169296 --4.620720,-0.098588,0.233216,0.708214,0.018850 --70.349128,-1.153332,-1.355448,-1.793892,-1.687344 -1.898174,-0.223202,-0.828497,-1.600904,1.489863 -18.909322,0.517659,-0.755383,0.186767,-0.725744 -57.884234,0.515048,1.135566,0.570891,3.852731 --7.367542,-0.234020,-0.075951,0.987335,0.466358 --127.809679,-2.635748,0.924270,0.327821,0.003376 --67.517171,-1.365824,1.796361,0.502784,-0.148969 -57.598113,1.103877,-0.589272,-0.002404,0.475168 -116.077897,2.403416,1.050654,0.201099,-0.057619 -32.961565,0.830336,-0.477657,0.071566,-0.856084 -2.406360,0.125225,0.543298,0.122298,-0.429406 -1.471337,-0.137449,1.314914,1.612278,0.952875 --33.271948,-0.488849,-1.768439,0.140886,-1.119617 -24.444454,0.548884,0.327880,-0.125454,-0.254956 -49.593564,1.088770,-1.171654,0.018819,-0.376301 --13.356460,-0.234153,0.767435,1.579213,-0.234137 --44.416671,-0.782003,0.471542,-1.771069,-0.759703 -41.170334,0.950424,0.491919,-0.898415,-0.576904 --5.254927,-0.230935,1.126565,1.848956,0.696206 --144.195252,-2.906988,-1.631276,-1.038544,-0.374822 -19.705454,0.500240,-0.876774,1.220821,-0.533600 --6.852548,-0.167118,-0.816936,1.206509,0.146714 --20.864428,-0.471038,-1.407464,-1.448084,0.232050 -8.425794,0.164761,0.243953,0.173342,0.050888 -12.293810,0.499864,-2.100027,0.367649,-1.399306 -85.806603,1.450928,-1.024029,-1.682659,1.807197 -89.358875,1.687142,1.479944,-0.007973,0.881640 --49.393801,-1.368315,0.105754,0.911363,1.987276 -46.981796,0.825416,0.021004,1.305479,0.813510 -85.879223,1.639965,-1.601966,0.075434,0.742127 -115.850708,2.319330,-0.309116,0.192049,0.393318 --14.166986,-0.040158,-0.681052,0.128104,-1.430775 -36.135083,0.594754,0.281191,0.758929,0.853416 -85.854880,1.612221,-0.891192,-0.268531,0.896839 -35.734975,0.727630,-0.080717,0.732640,0.051946 -0.229338,-0.194908,-2.940389,0.593557,1.133770 --28.979812,-0.730956,1.848609,0.685508,0.757922 -43.016920,0.951449,-0.924466,-1.703872,-0.366507 --13.907509,-0.623769,0.217433,-0.190682,1.914031 -61.430945,1.181891,-0.528297,-0.202524,0.480908 --42.942552,-0.927353,0.501094,0.975198,0.238369 --9.828396,-0.201492,0.668340,0.598794,-0.006521 --10.979241,-0.127655,2.052972,-2.929449,-0.560608 -86.784534,2.133033,0.588317,-0.151785,-1.952088 --23.105487,-0.473839,0.006422,0.546284,-0.014452 --22.910345,-0.358340,-0.181224,0.744192,-0.647542 -40.232111,0.857660,-1.002529,-0.019016,-0.159939 -63.271764,1.237654,-0.310308,-0.968046,0.379768 -15.946767,0.500917,0.751387,0.099332,-0.977555 --29.943037,-0.264515,-1.239258,0.635418,-2.003862 -40.601153,0.455888,0.927840,-0.643518,2.165002 -35.944347,0.570613,-0.612237,-0.331308,0.968185 -38.279295,0.519347,0.401712,-0.108760,1.532739 -17.189412,0.408607,-0.886681,-0.838586,-0.307808 -39.704834,0.915390,-0.635558,-0.116766,-0.549540 --8.449455,0.082440,-0.752156,-0.309209,-1.457551 --43.911474,-0.783766,-0.831822,-0.916192,-0.690541 --31.254562,-0.603981,-0.392471,-1.639484,-0.229549 --20.688623,-0.288549,-0.856772,1.734937,-0.783754 -1.668326,0.087047,-1.987569,0.091761,-0.299007 -20.361102,0.513600,-2.872262,-1.169917,-0.532701 --6.001539,0.242882,-0.548200,0.553149,-2.082099 -47.601730,0.619154,-0.728003,0.020794,2.057495 --68.389571,-1.615132,-0.810252,-0.734592,1.164739 -60.785446,1.047318,-0.043477,-0.229391,1.169590 --51.970801,-0.952316,0.830192,-0.772659,-0.676995 -30.600028,0.791032,-1.401851,1.402794,-0.909387 -10.552745,0.376410,1.125435,-0.869663,-0.902052 -11.954362,0.168655,1.410932,-1.090399,0.441941 -60.594607,0.964852,-0.592464,-1.234349,1.615583 -95.593864,1.964725,0.213980,-0.699726,0.035264 -16.927610,0.573744,0.292715,-1.288308,-1.276304 --32.906163,-0.627734,-2.487809,1.418531,-0.288039 -28.078212,0.386809,1.023710,2.012270,1.090980 -30.952623,0.696954,0.369642,1.173125,-0.333819 -48.821877,1.105900,0.078143,-0.816217,-0.563947 -38.691960,0.654976,0.420192,-0.993863,0.810799 --69.002925,-1.349576,0.111373,0.034027,-0.415214 -17.152741,0.714610,-0.336255,1.159074,-2.049941 --40.739696,-0.783253,-1.230864,0.813517,-0.322062 -38.276981,0.710960,1.159330,-0.360966,0.444263 --59.551267,-1.089633,-1.405567,-0.370508,-0.784762 --97.878485,-1.519346,0.551741,-0.451159,-2.832156 --25.790899,-0.564248,2.006093,1.542110,0.184551 --30.117274,-0.607354,0.418206,0.425844,-0.077221 -79.328950,1.617213,-1.330314,-0.898784,0.104356 --2.914316,0.175287,0.131928,-0.411823,-1.336725 -30.622220,0.576557,1.119575,3.078881,0.311250 -13.801043,-0.027515,-0.457096,1.661259,1.772252 --19.725101,-0.467701,0.477041,-0.046921,0.346504 -85.102098,1.608781,-0.462940,1.090862,0.828230 --1.948783,0.233786,0.833529,0.330880,-1.555896 --126.669663,-2.238231,0.457687,-0.606865,-2.120700 -99.095721,2.303639,1.146441,1.259233,-1.479444 -108.766963,2.015275,-1.102270,-0.397195,1.290644 --20.911083,-0.375196,0.557691,1.281644,-0.317715 -12.545889,0.573128,0.301107,-0.359630,-1.785866 --31.857777,-0.624345,-2.205566,-0.604745,-0.184525 --3.102783,0.202923,1.795878,1.547505,-1.515744 -39.098983,0.768207,3.926238,0.508269,0.215397 -51.680040,1.124353,-1.019664,0.564606,-0.334077 -42.990423,0.645376,0.686051,-0.964923,1.368632 -36.435469,0.534817,0.458387,-0.637308,1.228981 --43.498551,-0.808494,0.328751,0.915402,-0.501757 -21.371322,0.372105,1.223569,-0.053121,0.389159 --4.882158,0.032004,-0.677715,0.459972,-0.753418 -65.628407,0.963879,-1.369803,-0.557492,2.210523 --45.466160,-0.926930,-1.024388,-3.241267,-0.059525 --24.257059,-0.371858,1.653617,0.319525,-0.728462 --0.011753,-0.020902,-0.591571,1.277665,0.117327 --17.812900,-0.247177,-0.281100,-1.001620,-0.681984 -40.986422,0.695538,-0.071599,-0.293967,0.849102 -33.373605,0.949554,0.934320,-2.553921,-1.484898 -89.487862,1.991553,-0.147025,-0.550115,-0.832061 -42.524575,0.642723,0.709004,0.196521,1.329153 -38.099478,1.036088,-0.397558,-0.970124,-1.422975 --14.240897,-0.259042,-0.037222,-0.071601,-0.196350 --25.199474,-0.555547,-0.395681,-1.202150,0.204389 -11.504111,0.109395,0.223884,0.481009,0.725767 -31.779153,0.289775,-0.326024,0.871125,2.075401 -2.378890,-0.213443,-0.337086,0.148667,1.490726 -2.716467,0.048860,-0.662901,-0.701992,0.040592 --18.578302,-0.464404,-0.251539,0.783391,0.462061 -47.837256,1.096469,-0.202981,-1.319247,-0.625677 --43.511156,-0.787784,-0.472091,-0.168144,-0.620848 -96.318093,2.314659,-1.612716,0.686260,-1.867265 --101.660329,-2.499406,-1.645399,-1.389572,2.290943 -14.520997,0.444774,0.364140,0.149363,-0.825654 -26.928391,0.319175,0.115026,-1.875172,1.340450 --67.175480,-1.517174,-1.130069,-0.416194,0.750579 --6.523258,0.052258,-1.284584,-0.717016,-1.060605 -6.866515,-0.006071,-0.098890,0.081829,0.838491 -114.425274,2.124771,-1.222895,1.920368,1.331339 -36.080741,0.413435,-1.244655,-0.773789,1.876796 --19.136055,-0.246062,-0.175886,2.170943,-0.843247 --36.849172,-0.702053,-1.463515,-0.392108,-0.327662 -96.656003,1.749584,0.689708,-1.292263,1.381454 -19.912258,0.181063,-0.412414,0.586166,1.303278 -20.550135,0.267050,1.065480,0.082284,0.889631 -33.993085,0.175211,-0.313530,0.367482,2.985259 --48.815980,-0.881068,1.436335,1.186735,-0.712221 -18.950985,0.335058,-1.535572,0.469268,0.316156 -20.819823,0.425887,0.487872,-0.641487,0.019148 --13.547434,-0.066080,0.047399,-0.651836,-1.211016 --74.828752,-1.377669,0.513786,0.515035,-0.937825 --89.612221,-1.979300,0.239247,-1.072743,0.747910 --33.981594,-0.675178,-0.307962,-0.792420,-0.144519 --14.487640,-0.256180,0.479442,-0.061764,-0.241497 -12.843026,0.371146,-0.155677,0.086590,-0.603985 -40.147546,0.615771,-0.450189,-0.139446,1.203884 -28.438875,0.330851,-1.077376,0.879417,1.451001 -33.044619,0.778361,-0.003374,-0.818199,-0.551186 --7.231142,0.076822,-0.493757,0.996267,-1.282992 -22.902739,0.846771,-0.602821,-0.099094,-2.127227 -54.397143,0.897066,-1.137686,0.651522,1.274875 --16.394150,-0.522723,-1.408461,-0.704344,1.049009 --1.864271,0.097676,0.497998,0.024510,-0.773010 --13.743958,0.064474,-0.144088,-0.939335,-1.975467 --22.906456,-0.252568,-1.430141,1.632411,-1.247783 --86.353794,-2.084113,0.287329,-0.287448,1.724697 -70.123403,1.100960,0.496699,-1.229550,1.958347 -56.681980,1.196572,-0.933268,-0.027305,-0.158530 --4.234024,-0.073973,-1.385988,1.972542,-0.075666 --18.136725,-0.469474,-0.465730,-0.463418,0.542560 -91.693106,1.901191,-1.513714,-0.708407,-0.060661 -42.551831,0.840734,-0.532169,-1.292625,0.207803 --62.895569,-1.561493,-0.211667,-0.330120,1.503420 --0.223739,-0.054894,0.645216,0.521122,0.285554 --23.931246,-0.668090,-0.755745,-0.174960,0.992042 -12.520281,0.409763,0.697903,1.403370,-0.861088 --48.443983,-1.013896,0.255384,-0.925425,0.085687 --19.799870,-0.095464,-0.833056,-0.080600,-1.776246 -45.440854,0.840620,-0.366824,-0.238932,0.546734 --34.220464,-1.130204,0.602118,1.516394,2.411677 -48.852498,0.665417,-0.378008,-0.878323,1.941217 --93.505208,-1.993736,-1.209641,1.227669,0.374057 --40.707571,-0.856498,-1.793106,-1.264498,0.097670 --53.285959,-0.921860,0.069344,0.207267,-1.003957 --14.787532,0.085893,-0.851406,-0.229800,-2.219300 --61.815972,-1.515191,-0.249036,1.644968,1.366874 -25.991988,0.678947,-0.160360,-0.049029,-0.812402 -8.030046,0.260442,-1.962626,-1.008555,-0.538842 --25.117915,-0.735530,0.609138,1.091310,1.236093 -18.348504,0.536510,-0.009119,0.028181,-0.898468 -24.635405,0.436739,-1.071054,1.235782,0.404295 -24.708270,0.500666,-0.660321,0.007063,0.049774 --48.830849,-0.756351,-1.081548,-0.646573,-1.422254 -27.712862,0.555604,-0.151273,-0.197338,0.089581 --34.087315,-0.607875,0.833334,-0.548269,-0.539123 --90.022125,-1.893615,-0.817089,0.001205,0.213294 --21.329391,-0.589365,-0.692910,0.357015,0.849602 --22.272992,-0.319054,0.021312,1.076007,-0.796026 --21.632396,-0.531214,0.681360,-1.102292,0.483877 -36.754854,0.860473,0.282580,-0.167122,-0.583077 -5.886278,0.450603,-0.283139,-1.161784,-1.869816 --137.526774,-2.747505,1.388338,-0.526248,-0.499730 -33.575806,0.510792,0.854525,-0.657835,1.030581 -91.910281,1.855503,-1.611567,-0.346427,0.224235 --4.001870,-0.142147,0.872315,1.029457,0.338687 --54.219603,-1.111580,-1.567859,-0.531455,-0.035826 -4.873871,0.259723,-1.661520,0.638592,-0.904317 -114.941124,2.060748,0.971571,-0.248964,1.755341 --25.661291,-0.496932,-0.035008,-0.989958,-0.182569 --46.193889,-1.046911,0.718953,1.185704,0.536653 -20.810381,0.256030,1.014370,1.665474,0.982691 --45.015817,-0.860413,-0.576892,1.006293,-0.384556 -22.910711,0.496714,1.523030,0.647689,-0.138264 --33.032443,-0.612789,0.334457,0.285865,-0.387702 --34.961496,-0.795557,-2.216819,1.279519,0.424394 -50.668787,0.969457,1.775311,-0.646227,0.427194 --11.003652,-0.053905,2.320041,1.121858,-0.982304 -81.593908,1.738851,0.814152,1.832557,-0.321243 --32.493980,-0.573662,-0.543425,-0.032753,-0.546859 -64.193621,1.221034,-0.959439,-0.226484,0.582098 -42.167659,0.883110,3.193108,-0.180480,-0.077837 --27.201119,-0.718407,1.247742,-0.294950,0.894924 --81.926073,-1.866540,0.790626,-0.684630,1.007514 --11.309361,-0.158008,-1.654857,-1.012104,-0.426881 -35.624236,0.466671,1.747363,-0.948917,1.521006 --13.331116,-0.335138,0.853976,1.554160,0.342338 --43.482050,-1.021233,-0.564079,0.243801,0.708356 -62.833104,0.995582,-1.786375,-1.638029,1.703173 --32.481385,-0.675708,-1.085825,-1.087246,0.034152 --47.870125,-1.062394,0.985730,-0.187144,0.428307 -28.896078,0.379153,-2.390304,-0.596974,1.230222 --15.612306,-0.262660,0.462484,0.960305,-0.336387 -1.681588,0.072037,0.077481,-0.951918,-0.212209 --32.300419,-0.729602,0.572390,-1.293142,0.361417 -12.047486,0.055725,1.529550,-1.692465,1.094192 --39.403919,-0.827590,-2.921350,-1.072139,0.086144 --49.645544,-0.877983,0.367366,-0.226479,-0.826880 --23.020813,-0.544919,1.103302,-0.037635,0.399136 -30.937179,0.610216,0.224216,-0.586531,0.156973 -14.194066,0.143049,0.582571,0.692858,0.849612 -33.475474,0.655901,-0.388518,-0.018709,0.194736 --80.304555,-1.562546,-1.254289,0.794265,-0.529053 --37.190233,-0.694713,0.152355,-0.524088,-0.409282 -27.851444,0.521942,0.346448,0.250493,0.296985 -5.970137,0.209659,-0.973546,0.807123,-0.491636 -54.064252,1.263707,0.199810,0.543479,-0.846316 --14.248123,-0.245743,-0.054295,-2.696887,-0.272724 --1.581479,-0.042823,-2.246889,0.848431,0.058023 -21.017288,0.249309,0.023903,0.144888,1.045088 --59.163191,-1.006017,0.791663,1.158111,-1.214189 --6.308441,-0.278883,-1.405190,-0.048897,0.845158 --4.371930,-0.161286,0.174578,1.886186,0.404051 --1.301508,-0.189120,1.511155,-0.127549,0.921650 -10.228018,0.120296,-1.124642,0.711615,0.514439 --45.847015,-0.839722,-0.525755,-2.123896,-0.599393 --50.707960,-0.716822,-0.121748,-0.082681,-1.866537 --4.740566,-0.223463,-0.072829,0.473238,0.714000 --45.357775,-1.091934,0.553251,0.171839,0.890248 -29.547676,0.811397,1.537932,0.818778,-1.148263 -5.915717,0.324057,-2.991136,0.085209,-1.147691 -5.174835,0.133541,0.956702,0.708109,-0.152470 --44.991865,-0.828995,0.610370,0.747294,-0.560181 -37.898409,0.625508,0.123548,-0.592356,0.885231 --17.421907,-0.629263,-1.707358,-0.535801,1.533728 --26.878637,-0.391758,-0.322320,1.615376,-0.922410 --19.893022,-0.348652,2.076748,-0.321635,-0.349258 -8.776043,-0.025554,-0.370614,0.543600,1.172729 -37.225571,0.698223,0.635172,0.895193,0.393485 -70.103057,1.594505,-2.153390,-0.991392,-0.846961 --32.334315,-0.613403,0.170416,-0.388177,-0.302470 --55.259857,-1.004141,0.234215,-0.034685,-0.767798 --23.680551,-0.360039,0.312378,0.373349,-0.728077 --80.319976,-1.371901,-0.209324,1.471170,-1.613561 -15.409722,0.230071,0.421587,0.665924,0.497743 -2.267065,0.073197,-0.083718,1.191707,-0.150243 --49.916276,-0.785986,0.721563,0.303819,-1.381045 -12.871000,0.338496,2.270693,0.632782,-0.415288 --27.608034,-0.676922,0.931280,1.031000,0.611676 --15.877091,-0.194269,1.655407,1.048707,-0.755792 -51.914488,0.978422,-0.330789,-1.103670,0.522143 -47.207316,1.189470,0.701173,0.597400,-1.227608 --2.785427,0.177701,0.610586,0.380198,-1.335344 -68.111688,1.712040,0.475431,0.434770,-1.747637 -47.634653,0.766080,-0.203674,-0.100154,1.226933 -24.493869,0.522251,0.889037,2.272435,-0.097915 -30.764412,0.871297,1.938929,0.126380,-1.345980 --15.792703,-0.429244,0.462173,-1.598124,0.588553 -85.883753,1.804348,-1.293273,0.719758,-0.190904 -9.787300,0.096996,2.092387,-0.818221,0.595157 -52.017567,0.890198,-0.929511,-1.846188,1.035249 -0.410333,-0.134309,-1.748532,0.793489,0.810808 -11.528844,0.283288,-0.474904,0.586694,-0.258905 --6.574331,0.040919,-0.513214,0.740824,-1.002187 -49.733365,0.869156,0.037938,-0.763286,0.887291 --27.621629,-0.606700,-0.491902,1.200079,0.211284 -79.528706,1.464177,1.758620,0.280636,0.996859 --97.818389,-1.831682,1.857702,1.497432,-1.051310 -0.442045,-0.087435,-0.059415,0.949595,0.548320 -50.991665,0.812526,1.003533,-0.072010,1.356240 -19.494786,0.496199,1.935154,0.511500,-0.535317 -84.701319,1.672572,-0.055769,-0.705012,0.419019 -3.594766,0.349650,-0.263448,0.095344,-1.564803 -38.933752,0.882333,0.265878,-0.470042,-0.452090 --30.647922,-0.530501,-2.301921,-0.275052,-0.575818 --45.331845,-0.769142,-0.193826,0.829475,-0.939903 -3.364084,-0.049464,0.382410,-1.122722,0.674819 --89.400093,-1.975488,0.028458,-2.065083,0.751099 --49.491035,-0.785989,0.507991,-1.836205,-1.331233 -27.421077,0.476358,2.759660,1.060210,0.505470 -41.101274,0.938284,-0.462275,0.096121,-0.516045 -50.610184,0.953125,0.516178,0.725096,0.513085 --63.214225,-1.525656,2.558199,-0.551858,1.262584 -31.852687,0.722381,-0.399636,1.726964,-0.372833 --17.588273,-0.070166,0.207688,0.429618,-1.660961 --88.464104,-1.654887,0.093372,-0.122709,-0.960046 -5.260413,0.408253,0.472597,1.029156,-1.702584 --6.938688,-0.269407,0.074095,1.502357,0.717542 --96.661404,-2.018488,-0.122057,0.793176,0.145064 -62.456248,1.100821,0.361661,0.532006,1.061371 --9.371075,-0.067178,1.487246,0.682052,-0.715760 --73.494460,-1.778720,-0.055585,0.654366,1.496044 --71.901603,-1.103367,2.493000,0.388579,-2.152891 --42.354392,-0.956436,0.355613,1.484116,0.472406 -9.665231,0.191099,0.746254,-1.359856,0.046437 -44.867748,1.108183,-1.053416,0.612774,-1.039906 -21.183997,0.350630,1.109700,0.634721,0.489187 --25.204894,-0.602212,0.301792,-0.998385,0.468774 -21.433070,0.428317,-0.165631,-0.593811,0.077156 -12.227700,0.197292,1.071543,1.699957,0.311309 -14.413645,0.323168,-1.594703,-0.466037,-0.147603 -90.460920,1.954157,1.481663,-1.058908,-0.505747 -29.656326,0.551476,-1.326472,0.390696,0.340589 --50.240029,-0.985804,0.894647,1.194763,-0.284155 --30.211410,-0.714681,0.168060,0.321679,0.521280 -29.442950,0.977816,1.386349,1.153590,-2.105647 --34.312950,-0.790474,1.345420,1.882024,0.471468 --55.730419,-1.179369,1.528031,0.671531,0.172254 -11.596208,0.308051,-0.343192,0.269127,-0.391649 -18.862222,0.224685,-1.760809,-1.418366,0.932591 --0.132811,-0.045586,0.352055,-0.241236,0.243339 -34.288841,0.782874,0.550798,0.408053,-0.431126 --35.717533,-0.650643,-0.863991,-0.592394,-0.487125 -30.046681,0.690144,0.012592,0.224092,-0.401220 -40.830804,0.715381,0.019617,0.438476,0.718186 -31.759406,0.678931,-0.340908,-0.519396,-0.136995 --7.591012,0.071254,-0.918127,-0.695695,-1.293508 --0.826011,-0.172627,-1.576392,0.652323,0.883660 -43.874916,0.872321,-0.808298,2.189803,0.183342 -13.101358,0.771699,-1.739714,1.148766,-2.848543 -115.176759,2.165056,1.026986,0.212574,1.190549 -16.906448,0.088407,-0.193659,-1.141689,1.477530 --21.746485,-0.436720,-0.247026,2.099722,-0.066133 --53.415953,-1.010598,0.664927,1.530195,-0.515218 --48.434093,-1.143019,0.072874,-0.064689,0.820158 --2.631360,-0.023552,-0.369527,0.225308,-0.174354 --54.954697,-1.187598,0.413799,0.633777,0.309821 --23.473225,-0.465310,-1.502970,2.644343,-0.105948 -10.940720,0.181866,-0.849844,-0.459361,0.248221 -32.263143,0.418398,-1.503080,0.650450,1.401599 -25.359949,0.721672,0.489375,-0.524520,-1.129052 -20.613311,0.179894,-1.570501,0.918317,1.392002 --29.988207,-0.611769,-0.692421,-0.429302,-0.037037 -35.223990,0.638051,-0.542674,-1.801058,0.500844 --2.764751,0.085318,0.522514,-0.830444,-0.808266 --28.889546,-0.715304,0.216459,-0.730367,0.679598 -28.701139,0.491429,-0.098845,0.195482,0.569760 -19.284956,0.253165,0.454741,-0.033613,0.820344 --0.255055,0.214094,0.385317,0.173181,-1.245739 --11.664448,-0.034712,0.751933,1.142823,-1.168678 --42.782831,-0.825497,-0.563725,0.412931,-0.321386 -24.808421,0.547097,1.098777,-0.217681,-0.202193 --62.281194,-1.122545,0.271495,1.017661,-0.917503 --14.930170,-0.626967,-0.562467,0.707752,1.812449 -33.335632,0.736844,0.515939,0.066991,-0.281328 -8.069530,0.298753,1.148446,-0.426358,-0.751791 --4.743180,0.048522,-0.050238,0.270457,-0.830950 --28.537607,-0.622649,-1.320023,-0.742471,0.194607 --13.095268,-0.221254,0.815737,0.307407,-0.276813 --24.921538,-0.300860,0.251474,0.388979,-1.209477 -2.652468,0.293558,-0.035641,0.466430,-1.356582 --69.046695,-1.581191,0.146793,-0.483061,0.895038 -43.306319,1.073632,-0.700121,0.132970,-1.026515 --39.964274,-0.989628,-0.224633,-0.982487,0.940771 --19.340519,-0.731632,0.768840,0.049308,1.890441 --39.505916,-0.772825,0.081874,-0.485364,-0.236819 -42.097830,0.919076,0.321698,0.267392,-0.290275 --5.089397,-0.235970,0.266818,0.421546,0.744186 --144.881640,-3.019512,1.238946,1.800511,0.183850 -35.418594,0.838941,-1.100154,-0.558302,-0.617253 --7.335260,-0.112328,0.757508,0.614167,-0.220970 --14.755751,-0.132634,-0.120381,1.107081,-0.974529 --25.456229,-0.544383,0.375698,-1.150994,0.110923 -64.821941,1.276965,-1.075312,-1.207022,0.338023 --44.464477,-1.050141,0.695203,1.370536,0.757495 -11.624904,-0.035826,0.821903,-2.619745,1.564644 --56.422313,-1.090966,-0.214921,-1.105705,-0.410814 --103.513051,-2.077812,0.360648,1.643378,-0.320298 --61.073277,-1.200296,-0.653329,-0.474945,-0.334501 --13.053166,-0.517288,-0.362839,2.298898,1.409347 --99.305103,-1.831145,-0.086070,-2.075588,-1.228444 -13.044237,0.013002,2.720169,-0.264657,1.453534 --46.103766,-0.823935,0.000683,2.049812,-0.719109 -13.514751,0.323079,-1.563191,-0.291811,-0.252354 --53.548798,-1.209695,1.218762,1.530751,0.599929 -19.281787,0.259883,-1.320457,-1.236951,0.781823 -18.011152,0.235615,1.143754,-1.478586,0.770865 -0.469560,0.013929,-0.144360,0.198085,-0.024125 --10.787184,-0.328618,-0.162793,-0.544114,0.603187 -69.436797,1.376496,1.466541,0.687066,0.313132 --36.058708,-0.501784,-0.061004,0.322975,-1.372466 --10.080897,-0.150320,-1.172234,-1.042578,-0.326696 --2.106509,0.075805,-0.147057,0.975120,-0.677162 -41.746258,0.785800,-0.047711,-0.966976,0.425458 -46.924637,0.958386,1.074318,0.229075,0.051661 -19.423343,0.168461,1.139879,-1.006543,1.317598 --0.826030,-0.077102,0.827183,0.276691,0.341152 --8.281680,-0.308034,1.395684,1.310309,0.779661 --20.127015,-0.379128,-1.014757,-0.581681,-0.203580 --74.361561,-1.287164,1.038379,-0.583599,-1.397118 --45.163913,-0.709580,-0.308483,-0.217017,-1.258505 -39.981734,0.659246,-0.762725,-1.607560,0.937570 --43.504776,-0.985726,-0.792873,-0.530258,0.504047 --6.374632,-0.134017,0.648280,-0.784898,0.014688 --20.759861,-0.637387,-0.570746,1.420504,1.189017 -12.011867,-0.052404,-0.062004,1.246892,1.704106 --71.199152,-1.731201,0.443002,0.041326,1.494938 --56.686733,-1.193925,2.231300,1.732515,0.142943 --50.043678,-1.193637,-0.670620,1.000582,0.919154 --100.984519,-1.803140,0.508725,0.267127,-1.584136 -2.983288,0.020886,-0.616361,-1.260165,0.230701 -60.563058,1.275391,0.839469,0.533359,-0.151714 --11.089998,-0.450013,0.358454,-0.535335,1.257149 --26.635733,-0.487203,-1.296117,-0.769996,-0.351921 -30.426174,0.681953,-0.130143,0.324166,-0.310267 --22.101602,-0.436386,-0.370011,-0.088363,-0.109610 -78.792358,1.513450,1.854093,-1.024187,0.630812 --23.880608,-0.465806,-0.451659,-0.073948,-0.150838 --0.997821,-0.094459,1.279465,-0.863883,0.419610 -22.142600,0.357787,1.053802,1.083051,0.560785 --7.454662,0.014273,0.686318,-0.407036,-0.953939 --119.255005,-2.650970,-2.073390,1.246085,1.091507 --23.129794,-0.580523,0.394672,1.669905,0.588578 --48.371726,-1.251539,1.117296,-0.082151,1.443765 -17.597788,0.347562,0.191577,0.355669,0.086698 --10.067515,-0.003603,0.877362,1.503398,-1.158365 -1.111934,-0.026406,0.424061,-0.809604,0.280161 -57.111034,1.223083,-0.570351,0.352505,-0.258854 --77.053386,-1.776012,1.384532,0.252569,1.063941 -2.268242,0.271579,1.053153,-1.081057,-1.276749 --3.354548,0.045572,0.633919,2.143944,-0.651600 -33.512741,0.367287,-0.349317,-0.223466,1.838184 --52.433682,-0.763259,0.048085,-1.627542,-1.804882 --21.448359,-0.269875,0.377300,-0.444293,-0.978764 -4.519377,0.104201,-0.280675,-0.753965,-0.062593 --68.918673,-1.406317,0.760056,-1.504720,-0.083106 -10.127135,0.105894,-1.564242,1.976441,0.584413 --45.802438,-0.831955,-1.066235,-0.815376,-0.638283 --38.900006,-1.110576,1.271555,0.935678,1.752270 -24.078347,0.414866,0.558140,0.043515,0.463289 --57.660865,-1.119546,1.229215,0.178695,-0.393530 --84.828151,-1.656623,0.721479,-0.735303,-0.524448 --2.804653,-0.175564,1.295872,0.098068,0.668655 --33.665356,-0.712846,1.503993,-0.254977,0.106430 --14.578108,-0.327895,-0.867130,0.825098,0.155191 --55.093470,-0.799941,1.001632,-0.393053,-1.908002 -56.502928,1.163164,0.462103,-0.981509,0.010233 --76.823729,-1.351074,0.547265,0.481866,-1.322458 -38.265512,0.346710,-0.032281,-1.840078,2.511557 -85.418113,1.761188,0.449474,-1.181874,-0.000314 -95.220571,1.962587,1.341476,1.011463,0.003696 -11.855867,0.257550,-0.026514,-1.918771,-0.074446 --9.195330,-0.445503,-0.522860,1.579572,1.453384 --3.531133,-0.019420,-1.616311,0.799942,-0.303180 --98.141143,-2.172670,-0.090533,-0.535328,0.847422 -1.791711,0.196131,0.257976,-1.548961,-0.904068 --54.306016,-1.000331,0.179582,0.513908,-0.677745 --75.423846,-1.466785,0.515628,0.975312,-0.501402 --35.614170,-1.070852,0.189480,-1.280456,1.911419 -90.087932,2.056544,-0.926381,0.473632,-1.130888 -89.086373,1.765454,0.917862,-1.260884,0.404982 -14.686177,0.392416,-1.769076,-0.025574,-0.508964 --25.583447,-0.443044,0.463185,0.296576,-0.479493 --49.816583,-0.995148,0.178477,-2.177065,-0.181503 --16.832850,-0.597510,0.978858,1.738900,1.422370 -16.746760,0.231446,-0.872961,-0.215668,0.646490 --8.238905,-0.096624,0.608246,-0.945746,-0.415967 -72.026878,1.217159,-0.431620,0.998311,1.521316 -0.678036,-0.131473,-1.606577,-0.436764,0.826047 --36.344711,-0.832356,0.632932,-0.552223,0.471416 -63.757595,1.105526,-1.143005,0.638730,1.187030 --4.604388,0.241962,-0.562288,-1.724918,-1.913280 -19.537678,0.200569,0.061680,-1.015822,1.148637 --37.127281,-0.550537,-0.147780,-0.508140,-1.220712 -48.152810,0.950016,-0.196281,-1.532701,0.243005 --38.609401,-0.973379,-0.227476,0.336145,1.007133 --28.774276,-0.359769,0.260281,-0.413465,-1.326048 --17.707367,-0.709021,0.177750,-0.525880,1.953259 --93.405665,-1.951102,0.759712,-1.817979,0.143588 -28.893813,0.436560,-1.009731,-2.362932,0.903935 --3.692199,-0.351414,1.532368,-0.818429,1.563413 -1.813247,0.145521,1.588627,0.140983,-0.614123 --12.124965,-0.170185,0.955305,0.696387,-0.453228 --40.156480,-0.742484,1.685014,1.230875,-0.485306 -41.745142,1.009472,-0.091845,-1.499016,-0.844941 --10.095493,-0.185288,-0.147002,0.043811,-0.129821 -16.841777,0.225264,1.702515,-1.269330,0.692723 --82.451454,-1.594428,0.046981,0.005244,-0.599375 -3.901726,-0.039555,0.029756,0.028318,0.681501 -76.221557,1.551152,0.067518,1.179297,0.115675 --25.149857,-0.504180,-0.487229,0.347676,-0.081523 -11.160592,0.113517,-1.237815,1.586017,0.662131 --76.396711,-1.607661,0.006800,2.023606,0.184741 -0.323583,0.307802,0.743264,-1.348185,-1.710168 --47.042409,-1.179040,0.201160,-0.464617,1.187679 -21.987348,0.603095,0.264775,2.485190,-0.850535 -46.256643,0.899600,0.629629,0.812862,0.307300 --16.686025,-0.369277,0.408653,-1.776235,0.143388 -23.139777,0.679373,0.462591,0.666313,-1.148794 --54.205438,-1.195883,-0.609783,1.196631,0.444603 -39.793632,0.753342,0.673181,1.289753,0.381158 -31.355082,0.683329,2.363872,1.072978,-0.209314 --26.089039,-0.580053,0.472002,0.499685,0.239405 --81.013612,-1.388543,0.159463,-1.217283,-1.600271 -92.744917,2.074083,-0.286004,-2.530288,-0.919385 -36.381039,0.559790,0.459180,0.833922,1.080781 --48.392472,-1.320233,-0.469176,1.179440,1.831459 --41.943950,-0.649278,-0.769973,0.034083,-1.223940 -82.141220,1.613711,0.964087,-0.244157,0.453534 --21.799534,-0.176774,-0.284875,-0.491154,-1.548618 -49.220912,1.249505,-0.772569,-0.349439,-1.332777 -20.452186,0.622207,-0.075764,1.039112,-1.138833 -5.160095,0.000528,-2.296181,-1.443855,0.601207 --12.192105,-0.412221,0.428186,0.537630,0.913474 --17.792743,-0.322680,0.556230,1.328194,-0.250833 -20.458890,0.522835,2.142270,-0.024355,-0.573700 --67.468406,-1.044809,-1.103208,2.056207,-1.966357 -92.613380,1.673452,0.829732,-1.299581,1.340461 -3.765963,-0.019638,1.364140,0.223914,0.552490 --22.417147,-0.643550,-0.403648,-0.334775,1.029961 --80.232568,-1.542802,-0.634286,0.596441,-0.632755 -30.360086,0.679143,0.731871,-0.329448,-0.302045 -7.814141,0.160574,1.190646,0.436938,0.003046 --62.647985,-1.395789,-0.195390,-0.723990,0.591349 --20.226385,-0.440044,-1.435862,1.441273,0.130741 -80.098095,1.362563,-1.123494,3.152057,1.640615 --38.333188,-0.975873,2.632382,-0.949399,1.053642 -13.985381,0.326927,-2.211135,0.829406,-0.219101 -55.172745,0.975817,0.054934,-1.245717,0.918454 -16.132126,0.301700,-0.890318,-1.843794,0.175536 --22.305862,-0.128538,0.092845,-0.548725,-1.881849 -26.990770,0.621572,-0.029263,0.377100,-0.369610 --73.141828,-1.379319,1.794558,-0.033127,-0.730930 --14.368907,0.078577,-0.979721,0.458168,-2.128734 -80.426368,1.846637,-0.691908,-1.525525,-1.070085 --58.909457,-0.971657,0.862393,-0.626717,-1.379618 -118.026988,2.526932,1.044161,-0.489439,-0.530869 -8.463238,0.015618,1.519284,-0.907756,0.902277 -19.374998,0.403730,0.324359,-0.903702,-0.024196 --72.240465,-1.415371,-0.802277,-0.342715,-0.420645 -1.213852,-0.230866,0.692919,-1.338606,1.453261 -37.280494,0.738467,-0.301104,-0.115648,0.171368 -123.353110,1.995667,-0.183197,0.606723,3.109919 -106.240231,1.904137,-0.159045,2.218780,1.625959 -43.192714,0.932192,-1.106294,1.135620,-0.236555 -15.588410,0.556553,-1.547304,0.486036,-1.335482 --93.738805,-1.870792,1.676437,0.018418,-0.351513 -32.118610,0.640543,-1.299216,-0.113128,0.123078 -5.059099,0.057013,0.507836,1.528468,0.268592 -62.874109,1.317115,-0.607822,-2.121855,-0.118069 -94.540097,1.865775,0.656554,-1.191303,0.473833 -55.509165,1.041725,0.579857,-0.129517,0.583544 -17.820893,0.464370,-0.885249,0.316358,-0.550552 --15.158440,-0.342549,-0.206611,-0.962733,0.170464 -13.822799,0.399688,0.586364,-0.528617,-0.651357 --34.908483,-0.734174,1.455822,0.457280,0.081996 --86.489070,-2.081929,-0.096713,0.211017,1.696456 --17.171242,-0.138456,-0.850520,-0.209023,-1.224298 --26.725651,-0.516430,-0.746578,-0.205859,-0.196467 --19.793182,-0.342688,-0.777817,-1.407512,-0.371441 --21.808750,-0.279810,-0.728272,0.505966,-0.964538 -57.903675,1.170199,0.554058,0.079598,0.134296 --54.266640,-1.065114,-0.186971,-0.609512,-0.305225 -48.680969,0.953137,0.361504,-0.612437,0.287124 -7.893616,0.265362,-0.134279,-2.438817,-0.582759 -26.717704,0.428817,-0.367028,0.176442,0.693106 -0.010860,0.072681,-0.037571,-0.089234,-0.411493 -73.653356,1.393455,0.379640,0.429341,0.710549 -72.191436,1.390208,-1.311836,0.010353,0.557810 --41.673025,-0.611518,-1.351685,-0.923233,-1.406661 -94.768012,2.075261,0.197911,1.735964,-0.689188 -28.839738,0.756989,1.355638,0.869606,-0.922165 --24.826396,-0.479174,-1.196207,-1.106335,-0.185659 -71.859667,1.446978,-1.485560,1.031845,0.196555 -11.100262,0.225762,1.303508,-0.975366,0.017613 -69.434230,1.633432,-0.754276,0.302635,-1.146345 -94.072600,1.870965,0.534629,-0.868293,0.389614 --17.095965,-0.399899,0.182501,-0.044893,0.269295 -16.301071,0.253740,0.097983,1.085786,0.467693 -21.488666,0.683569,0.261251,1.211944,-1.365956 -12.046548,0.160018,1.448499,1.117399,0.501783 -32.130618,0.552300,0.582027,-0.696772,0.625631 -36.315323,1.058424,-2.039232,-1.183259,-1.758739 -30.287068,0.693479,0.617006,1.059936,-0.392013 --35.727477,-0.651418,0.424166,-0.320347,-0.483886 -92.664022,1.797687,0.572583,-0.571179,0.640843 --20.573242,-0.456121,-0.557423,0.595029,0.181427 -4.827582,0.232787,1.194592,-2.189922,-0.756764 -10.026700,0.075955,-1.237662,0.482249,0.742682 --3.363153,-0.018513,-0.827231,0.322719,-0.288659 -24.890289,0.399223,1.573987,-0.483186,0.647196 --26.477548,-0.780199,0.893698,-1.196789,1.330575 -32.745331,0.541463,-2.591042,-0.576510,0.759155 -1.609208,0.343618,-0.385082,0.324084,-1.763040 -6.716008,0.170865,0.347582,0.018434,-0.183983 -6.411841,0.194384,-0.295401,0.338484,-0.353166 --42.707499,-0.955123,-1.067533,2.062525,0.423599 --46.440364,-1.012831,-1.412304,-0.908024,0.314247 --57.549911,-1.304470,-0.939880,0.366598,0.669673 -15.291934,0.139887,0.157761,-1.390222,0.996118 --19.624005,-0.382817,-1.184031,-0.397680,-0.123729 --5.305926,-0.179093,0.734638,0.692856,0.395820 --41.537671,-0.861703,0.876456,-2.152384,0.030031 --12.269927,-0.190039,1.372848,-0.180392,-0.357445 --70.949815,-1.154363,0.158053,-0.389924,-1.751829 --27.343957,-0.487606,-0.420984,0.394452,-0.432558 --45.842871,-1.092164,-1.545730,0.913772,0.834751 --51.125283,-0.936506,-0.187329,0.292193,-0.667780 -76.643696,1.677701,1.628397,0.568983,-0.553588 -75.767908,1.399355,-0.646937,0.059630,0.924634 -46.433123,0.870068,0.364961,0.150419,0.495682 --13.251291,0.078635,0.346488,0.916328,-1.998201 -23.983712,0.695402,-0.804966,-0.111509,-1.141009 --45.187961,-0.824514,-1.056558,-0.528978,-0.608590 -66.675817,1.550500,-0.213989,0.984322,-0.998354 --17.470983,0.007178,2.212531,0.160490,-2.086478 -50.250749,0.963376,1.896793,0.822060,0.412781 -32.173526,0.706079,1.230001,-1.026369,-0.242681 --33.495466,-0.721738,-0.271655,-0.546680,0.176821 -37.466590,0.779349,-2.025808,-0.417910,-0.039020 -23.365530,0.670481,0.817890,-1.553759,-1.071857 --2.032212,-0.335785,-1.503143,-0.259591,1.669022 --22.491652,-0.268889,0.059218,2.573360,-1.106526 -49.108338,0.658544,-0.798297,-0.176947,2.010205 --100.213687,-1.840874,0.026091,-0.624819,-1.279577 -38.978823,0.632408,-1.570225,0.621810,0.972554 -58.812362,1.167782,-0.411877,0.337603,0.254421 --70.323806,-1.556629,1.754794,-1.280429,0.606010 --6.507764,-0.104449,1.161878,0.070052,-0.168822 -53.405935,1.296063,-0.151299,-0.365506,-1.107157 --63.584408,-1.372859,0.046935,-1.978225,0.351480 -17.749661,-0.088282,1.639117,-0.803675,2.579709 -19.786853,0.562881,-0.530971,1.987061,-0.879814 -67.989854,1.580541,-1.257578,-0.810858,-1.015094 -18.601461,0.224340,1.311984,-0.297482,0.904018 --76.668745,-1.517874,0.575205,0.890383,-0.357029 --36.668740,-0.718444,1.475356,0.310908,-0.213447 -12.554325,0.317453,-0.529332,-0.093387,-0.332860 --19.313369,-0.549557,-0.045411,-0.387131,0.859588 --31.555644,-0.482744,1.624678,0.122670,-0.953329 --24.432405,-0.509807,-0.658248,-2.488327,0.034441 --38.242766,-0.581782,2.067906,-0.197938,-1.173880 --60.224853,-1.053682,1.710613,0.950308,-1.067803 -47.524552,0.900399,-1.178041,1.183494,0.451225 --54.918400,-0.963759,-0.048652,0.343788,-0.957151 -14.522086,0.368673,1.278452,0.028745,-0.393339 -25.567486,0.249384,0.279022,-0.095296,1.577453 --4.583559,0.281009,-0.158154,1.012637,-2.132596 --125.221331,-2.529560,-1.151014,0.241322,-0.296607 --19.579552,-0.228600,-0.191028,-2.562334,-0.994349 -21.021767,0.613518,-1.668584,-0.257377,-1.022793 --35.466411,-0.637740,-0.555477,-0.623141,-0.530997 --42.290382,-0.638962,1.009817,1.642015,-1.323090 --28.661548,-0.451949,-0.215610,-0.007045,-0.789339 --110.508705,-2.151815,-0.987180,-0.211130,-0.719153 -42.497611,1.051948,0.250200,-0.383971,-0.998061 -105.389371,2.281652,-1.880010,-1.535040,-0.617642 --1.380709,-0.107302,-1.126767,-1.570876,0.447717 --29.056470,-0.443282,1.711708,-0.172946,-0.884803 -17.215947,0.320786,0.286956,-1.270061,0.194052 -44.953433,1.195047,0.377212,-0.558922,-1.523187 -75.569458,1.614779,-1.406815,1.325432,-0.322028 -36.914028,0.506241,-1.049655,0.568103,1.447306 --22.989535,-0.186896,0.777407,1.205995,-1.630473 --25.522945,-0.606503,-0.694600,-0.459090,0.455904 -20.543404,0.400481,-1.195294,-0.077443,0.131065 -16.591857,0.296120,-0.234587,0.005113,0.261055 --57.255754,-1.317132,-0.752435,-1.002217,0.776028 -23.671733,0.264020,0.288724,0.732492,1.272354 --14.217121,-0.319848,2.089539,-0.835143,0.151758 -56.712634,1.428858,0.329107,-0.585928,-1.474132 -6.823869,0.114228,-0.056946,-0.363612,0.150302 -67.204299,1.628616,-0.055548,-1.703382,-1.380101 -37.489672,0.698828,-0.526035,0.492019,0.420973 --0.303095,-0.064138,0.421921,0.321357,0.328762 -18.408561,0.550052,-1.334025,0.105376,-0.968344 --11.155907,-0.238110,1.172757,-0.904614,0.046003 -1.472200,-0.064078,-0.162124,-2.196128,0.536291 -41.846845,0.996048,1.501334,-1.421811,-0.756795 -54.259315,1.200262,1.153901,-0.411427,-0.463161 -73.233162,1.676393,1.137315,-1.153351,-0.945507 -6.604370,0.091752,0.214713,-0.116229,0.252240 --38.915377,-0.892788,-1.968504,0.034714,0.513614 -12.030480,0.361636,1.538037,0.361396,-0.645120 -10.685507,0.123205,1.695051,0.043602,0.551485 -12.614775,0.185176,1.278866,0.222290,0.425445 -49.543400,0.887504,0.388312,0.060086,0.760847 --30.998905,-0.680025,-0.714351,0.293072,0.232254 --8.754455,-0.372207,1.543244,1.884586,1.088749 --13.589576,-0.471932,-1.077745,0.064280,1.088951 -6.690391,-0.222475,-0.403076,-0.674934,2.046862 -4.528719,0.199060,-0.385314,0.069802,-0.600217 --10.487547,-0.220964,-2.041735,0.208383,0.026886 -47.107226,1.192735,0.590422,0.731380,-1.257868 -1.142709,0.216776,-0.837262,-0.588867,-1.097303 From 9a4ac8853658b578c2f12f2d2d814c0fec8a5771 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 08:52:30 +0000 Subject: [PATCH 83/97] move position file when position set in Dataset --- tests/python_package_test/test_engine.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 91dcb827292e..f0252cf8452b 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -9,7 +9,7 @@ import re from os import getenv from pathlib import Path -from shutil import copyfile +from shutil import copyfile, move import numpy as np import psutil @@ -783,10 +783,11 @@ def test_ranking_with_position_information(tmp_path): assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) - copyfile(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) + move(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] + move(str(tmp_path / 'rank.train.position'), str(tmp_path / '_rank.train.position')) # test setting positions through set_position lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) @@ -800,8 +801,9 @@ def test_ranking_with_position_information(tmp_path): np.testing.assert_array_equal(positions_from_get, positions) # add extra row to position file + move(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) with open(str(tmp_path / 'rank.train.position'), 'a') as file: - file.write('pos_1000') + file.write('pos_1000\n') file.close() lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] From 74b693469ed95ff8ba825bd48c021eb01a1a4904 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 09:23:55 +0000 Subject: [PATCH 84/97] warn when positions are overwritten --- src/io/metadata.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/io/metadata.cpp b/src/io/metadata.cpp index 78414f445ee0..1fc47c46787f 100644 --- a/src/io/metadata.cpp +++ b/src/io/metadata.cpp @@ -531,9 +531,13 @@ void Metadata::SetPosition(const data_size_t* positions, data_size_t len) { Log::Fatal("Positions in learning to rank is not supported in CUDA version yet."); #endif // USE_CUDA if (num_data_ != len) { - Log::Fatal("Positions size (%i) doesn't match data size (%i)", len, num_data_); + Log::Fatal("Positions size (%i) doesn't match data size (%i)", len, num_data_); + } + if (positions_.empty()) { + positions_.resize(num_data_); + } else { + Log::Warning("Overwritting positions in dataset."); } - if (positions_.empty()) { positions_.resize(num_data_); } num_positions_ = num_data_; position_load_from_file_ = false; @@ -605,7 +609,7 @@ void Metadata::LoadPositions() { if (reader.Lines().empty()) { return; } - Log::Info("Loading positions..."); + Log::Info("Loading positions from %s ...", position_filename.c_str()); num_positions_ = static_cast(reader.Lines().size()); positions_ = std::vector(num_positions_); position_ids_ = std::vector(); From 59c275f7bf76f5a89ec5bfad22b7c2b87ea4e314 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Tue, 8 Aug 2023 10:23:41 +0000 Subject: [PATCH 85/97] skip ranking with position test in cuda --- tests/python_package_test/test_engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index f0252cf8452b..9481d8398209 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -748,6 +748,7 @@ def test_ranking_prediction_early_stopping(): np.testing.assert_allclose(ret_early, ret_early_more_strict) +@pytest.mark.skipif(getenv('TASK', '') == 'cuda', reason='Positions in learning to rank is not supported in CUDA version yet') def test_ranking_with_position_information(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' params = { From 8bbbc43a9894fcee7b4d7377eb44a9f03e6447fc Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Wed, 9 Aug 2023 05:50:17 +0000 Subject: [PATCH 86/97] split test case --- tests/python_package_test/test_engine.py | 74 +++++++++++++++++------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 9481d8398209..c60ba26e5b04 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -20,7 +20,7 @@ from sklearn.model_selection import GroupKFold, TimeSeriesSplit, train_test_split import lightgbm as lgb -from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame +from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series from .utils import (SERIALIZERS, dummy_obj, load_breast_cancer, load_digits, load_iris, logistic_sigmoid, make_synthetic_regression, mse_obj, pickle_and_unpickle_object, sklearn_multiclass_custom_objective, @@ -749,7 +749,7 @@ def test_ranking_prediction_early_stopping(): @pytest.mark.skipif(getenv('TASK', '') == 'cuda', reason='Positions in learning to rank is not supported in CUDA version yet') -def test_ranking_with_position_information(tmp_path): +def test_ranking_with_position_information_with_file(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' params = { 'objective': 'lambdarank', @@ -766,16 +766,56 @@ def test_ranking_with_position_information(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - positions = [] - with open(str(tmp_path / '_rank.train.position'), "w") as out_file: + with open(str(tmp_path / 'rank.train.position'), "w") as out_file: np.random.seed(0) for _ in range(lgb_train.num_data()): position = np.random.randint(28) - positions.append(position) out_file.write(f"pos_{position}\n") + + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + assert gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] + + # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned + assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'], 0.02) + + # add extra row to position file + with open(str(tmp_path / 'rank.train.position'), 'a') as file: + file.write('pos_1000\n') + file.close() + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + with pytest.raises(lgb.basic.LightGBMError, match="Positions size \(3006\) doesn't match data size"): + lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + + +@pytest.mark.skipif(getenv('TASK', '') == 'cuda', reason='Positions in learning to rank is not supported in CUDA version yet') +def test_ranking_with_position_information_with_dataset_constructor(tmp_path): + rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' + params = { + 'objective': 'lambdarank', + 'verbose': -1, + 'eval_at': [3], + 'metric': 'ndcg' + } + copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) + copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) + copyfile(str(rank_example_dir / 'rank.test'), str(tmp_path / 'rank.test')) + copyfile(str(rank_example_dir / 'rank.test.query'), str(tmp_path / 'rank.test.query')) + + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + + positions = [] + np.random.seed(0) + for _ in range(lgb_train.num_data()): + position = np.random.randint(28) + positions.append(position) positions = np.array(positions) - # test setting positions through Dataset constructor + # test setting positions through Dataset constructor with numpy array lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params, position=positions) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) @@ -783,12 +823,12 @@ def test_ranking_with_position_information(tmp_path): # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) - lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) - move(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) - lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] - gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] - move(str(tmp_path / 'rank.train.position'), str(tmp_path / '_rank.train.position')) + if PANDAS_INSTALLED: + # test setting positions through Dataset constructor with pandas Series + lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params, position=pd_Series(positions)) + lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] + gbm_unbiased_pandas_series = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) + assert gbm_unbiased.best_score['valid_0']['ndcg@3'] == gbm_unbiased_pandas_series.best_score['valid_0']['ndcg@3'] # test setting positions through set_position lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) @@ -801,16 +841,6 @@ def test_ranking_with_position_information(tmp_path): positions_from_get = lgb_train.get_position() np.testing.assert_array_equal(positions_from_get, positions) - # add extra row to position file - move(str(tmp_path / '_rank.train.position'), str(tmp_path / 'rank.train.position')) - with open(str(tmp_path / 'rank.train.position'), 'a') as file: - file.write('pos_1000\n') - file.close() - lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) - lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] - with pytest.raises(lgb.basic.LightGBMError, match="Positions size \(3006\) doesn't match data size"): - lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - def test_early_stopping(): X, y = load_breast_cancer(return_X_y=True) From e843de672ccf695f0880add853e5b9568e3bcda1 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Wed, 9 Aug 2023 05:52:40 +0000 Subject: [PATCH 87/97] remove useless import --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index c60ba26e5b04..ba74074d5d77 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -9,7 +9,7 @@ import re from os import getenv from pathlib import Path -from shutil import copyfile, move +from shutil import copyfile import numpy as np import psutil From 5efb9a9d5dceb07a9134c611df0e21e684101975 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Fri, 25 Aug 2023 01:55:45 -0700 Subject: [PATCH 88/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 82 ++++++++++++++++++++---- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index ba74074d5d77..e2dbf0303f62 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -747,7 +747,68 @@ def test_ranking_prediction_early_stopping(): with pytest.raises(AssertionError): np.testing.assert_allclose(ret_early, ret_early_more_strict) - +# Simulates position bias for a given ranking dataset. +# The ouput dataset is identical to the input one with the exception for the relevance labels. +# The new labels are generated according to an instance of a cascade user model: +# for each query, the user is simulated to be traversing the list of documents ranked by a baseline ranker +# (in our example it is simply the ordering by some feature correlated with relevance, e.g., 34) +# and clicks on that document (new_label=1) with some probability 'pclick' depending on its true relevance; +# at each position the user may stop the traversal with some probability pstop. For the non-clicked documents, +# new_label=0. Thus the generated new labels are biased towards the baseline ranker. +# The positions of the documents in the ranked lists produced by the baseline, are returned. +def simulate_position_bias(file_dataset_in, file_query_in, file_dataset_out, baseline_feature): + # a mapping of a document's true relevance (defined on a 5-grade scale) into the probability of clicking it + def get_pclick(label): + if label == 0: + return 0.4 + elif label == 1: + return 0.6 + elif label == 2: + return 0.7 + elif label == 3: + return 0.8 + else: + return 0.9 + # an instantiation of a cascade model where the user stops with probability 0.2 after observing each document + pstop = 0.2 + + f_dataset_in = open(file_dataset_in, 'r') + f_dataset_out = open(file_dataset_out, 'w') + random.seed(10) + positions_all = [] + for line in open(file_query_in): + docs_num = int (line) + lines = [] + index_values = [] + positions = [0] * docs_num + for index in range(docs_num): + features = f_dataset_in.readline().split() + lines.append(features) + val = 0.0 + for feature_val in features: + feature_val_split = feature_val.split(":") + if int(feature_val_split[0]) == baseline_feature: + val = float(feature_val_split[1]) + index_values.append([index, val]) + index_values.sort(key=lambda x: -x[1]) + stop = False + for pos in range(docs_num): + index = index_values[pos][0] + new_label = 0 + if not stop: + label = int(lines[index][0]) + pclick = get_pclick(label) + if random.random() < pclick: + new_label = 1 + stop = random.random() < pstop + lines[index][0] = str(new_label) + positions[index] = pos + for features in lines: + f_dataset_out.write(' '.join(features) + '\n') + positions_all.extend(positions) + f_dataset_out.close() + return positions_all + @pytest.mark.skipif(getenv('TASK', '') == 'cuda', reason='Positions in learning to rank is not supported in CUDA version yet') def test_ranking_with_position_information_with_file(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' @@ -757,7 +818,8 @@ def test_ranking_with_position_information_with_file(tmp_path): 'eval_at': [3], 'metric': 'ndcg' } - copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) + # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory + positions = simulate_position_bias(str(rank_example_dir / 'rank.train'), str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train'), baseline_feature=34) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) copyfile(str(rank_example_dir / 'rank.test'), str(tmp_path / 'rank.test')) copyfile(str(rank_example_dir / 'rank.test.query'), str(tmp_path / 'rank.test.query')) @@ -766,19 +828,17 @@ def test_ranking_with_position_information_with_file(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - with open(str(tmp_path / 'rank.train.position'), "w") as out_file: - np.random.seed(0) - for _ in range(lgb_train.num_data()): - position = np.random.randint(28) - out_file.write(f"pos_{position}\n") + f_positions_out = open(str(tmp_path / 'rank.train.position'), 'w') + for pos in positions: + f_positions_out.write(str(pos) + '\n') + f_positions_out.close() lgb_train = lgb.Dataset(str(tmp_path / 'rank.train'), params=params) lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased_with_file = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - assert gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] == gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] - - # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned - assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'], 0.02) + + # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias + assert gbm_baseline.best_score['valid_0']['ndcg@3'] + 0.03 <= gbm_unbiased_with_file.best_score['valid_0']['ndcg@3'] # add extra row to position file with open(str(tmp_path / 'rank.train.position'), 'a') as file: From ee2fc4befe569eb2d76d7a224164b90e094c180f Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Fri, 25 Aug 2023 03:10:05 -0700 Subject: [PATCH 89/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index e2dbf0303f62..c9376c1a3fb6 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -816,8 +816,12 @@ def test_ranking_with_position_information_with_file(tmp_path): 'objective': 'lambdarank', 'verbose': -1, 'eval_at': [3], - 'metric': 'ndcg' + 'metric': 'ndcg', + 'bagging_fraction': 0.9, + 'min_data_in_leaf': 50, + 'min_sum_hessian_in_leaf': 5.0 } + # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory positions = simulate_position_bias(str(rank_example_dir / 'rank.train'), str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train'), baseline_feature=34) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) @@ -857,9 +861,14 @@ def test_ranking_with_position_information_with_dataset_constructor(tmp_path): 'objective': 'lambdarank', 'verbose': -1, 'eval_at': [3], - 'metric': 'ndcg' + 'metric': 'ndcg', + 'bagging_fraction': 0.9, + 'min_data_in_leaf': 50, + 'min_sum_hessian_in_leaf': 5.0 } - copyfile(str(rank_example_dir / 'rank.train'), str(tmp_path / 'rank.train')) + + # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory + positions = simulate_position_bias(str(rank_example_dir / 'rank.train'), str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train'), baseline_feature=34) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) copyfile(str(rank_example_dir / 'rank.test'), str(tmp_path / 'rank.test')) copyfile(str(rank_example_dir / 'rank.test.query'), str(tmp_path / 'rank.test.query')) @@ -868,11 +877,6 @@ def test_ranking_with_position_information_with_dataset_constructor(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_baseline = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - positions = [] - np.random.seed(0) - for _ in range(lgb_train.num_data()): - position = np.random.randint(28) - positions.append(position) positions = np.array(positions) # test setting positions through Dataset constructor with numpy array @@ -880,8 +884,8 @@ def test_ranking_with_position_information_with_dataset_constructor(tmp_path): lgb_valid = [lgb_train.create_valid(str(tmp_path / 'rank.test'))] gbm_unbiased = lgb.train(params, lgb_train, valid_sets = lgb_valid, num_boost_round=50) - # the performance of the baseline LambdaMART should not differ much from the case when positions are randomly assigned - assert gbm_baseline.best_score['valid_0']['ndcg@3'] == pytest.approx(gbm_unbiased.best_score['valid_0']['ndcg@3'], 0.02) + # the performance of the unbiased LambdaMART should outperform the plain LambdaMART on the dataset with position bias + assert gbm_baseline.best_score['valid_0']['ndcg@3'] + 0.03 <= gbm_unbiased.best_score['valid_0']['ndcg@3'] if PANDAS_INSTALLED: # test setting positions through Dataset constructor with pandas Series From 98ac42de6662fbe6140a52d742752ecc469189d5 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Fri, 25 Aug 2023 04:41:27 -0700 Subject: [PATCH 90/97] Update test_engine.py --- tests/python_package_test/test_engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index c9376c1a3fb6..d0c06750c672 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -817,6 +817,7 @@ def test_ranking_with_position_information_with_file(tmp_path): 'verbose': -1, 'eval_at': [3], 'metric': 'ndcg', + 'bagging_freq': 1, 'bagging_fraction': 0.9, 'min_data_in_leaf': 50, 'min_sum_hessian_in_leaf': 5.0 @@ -862,6 +863,7 @@ def test_ranking_with_position_information_with_dataset_constructor(tmp_path): 'verbose': -1, 'eval_at': [3], 'metric': 'ndcg', + 'bagging_freq': 1, 'bagging_fraction': 0.9, 'min_data_in_leaf': 50, 'min_sum_hessian_in_leaf': 5.0 From ca4bd047790ab0d536f2d3e07da79c39fd3b0622 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Sat, 26 Aug 2023 01:39:25 -0700 Subject: [PATCH 91/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index ebb1dd919044..5502e7bac904 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -82,17 +82,33 @@ Support for Position Bias Treatment ------------------------------------ Often the relevance labels provided in Learning-to-Rank tasks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user. -LightGBM can make use of positional data, for example provided in an additional file like the following: +LightGBM can make use of positional data. + +For example, consider the case where you expect that the first 3 results from a search engine will be visible in users' browsers without scrolling, and all other results for a query would require scrolling. + +LightGBM could be told to account for the position bias from results being "above the fold" by providing a ``positions`` array encoded as follows: :: - 3 - 4 + 0 + 0 + 0 + 1 + 1 + 0 + 0 + 0 1 ... -It means the position of the document from the first data row, when presented to a user, was encoded as ``3``, second was encoded as ``4``, and so on. -A position could be any arbitrary integer number (not necessarily from a contiguous range) and could encode the presentation of an item even in complex layouts like 2D or consisting of several verticals (it is up to the LightGBM user to perform such encoding/mapping into integers). +Where ``0 = "above the fold"`` and ``1 = "requires scrolling"``. +The specific value are not important, as long as they are consistent across all observations in the training data. +An encoding like ``100 = "above the fold"`` and ``17 = "requires scrolling"`` would result in exactly the same trained model. + +In that way, ``positions`` in LightGBM's API are similar to a categorical feature. +Just as with non-ordinal categorical features, an integer representation is just used for memory and computational efficiency... LightGBM does not care about the absolute or relative magnitude of the values. + +Unlike a categorical feature, however, ``positions`` are used to adjust the target to reduce the bias in predictions made by the trained model. The position file corresponds with training data file line by line, and has one position per line. And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file. In this case, LightGBM will load the position file automatically if it exists. The positions can also be specified through the ``Dataset`` constructor when using Python API. If the positions are specified in both approaches, the ``.position`` file will be ignored. From 56a337f13fb6eec95e2edce2c9f1c6ffb607c228 Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Sat, 26 Aug 2023 01:57:36 -0700 Subject: [PATCH 92/97] Update Parameters.rst --- docs/Parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Parameters.rst b/docs/Parameters.rst index 0e85091d51cb..7d825f9f135a 100644 --- a/docs/Parameters.rst +++ b/docs/Parameters.rst @@ -1137,7 +1137,7 @@ Objective Parameters - separate by ``,`` -- ``lambdarank_position_bias_regularizer`` :raw-html:`🔗︎`, default = ``0.0``, type = double, constraints: ``lambdarank_position_bias_regularizer >= 0.0`` +- ``lambdarank_position_bias_regularization`` :raw-html:`🔗︎`, default = ``0.0``, type = double, constraints: ``lambdarank_position_bias_regularization >= 0.0`` - used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. From 2b88856993196d0936992fd5bae527c82a1aac2f Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Sat, 26 Aug 2023 02:01:34 -0700 Subject: [PATCH 93/97] Update rank_objective.hpp --- src/objective/rank_objective.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/objective/rank_objective.hpp b/src/objective/rank_objective.hpp index 5846e1b0092c..6bd5324812f8 100644 --- a/src/objective/rank_objective.hpp +++ b/src/objective/rank_objective.hpp @@ -27,7 +27,7 @@ class RankingObjective : public ObjectiveFunction { explicit RankingObjective(const Config& config) : seed_(config.objective_seed) { learning_rate_ = config.learning_rate; - position_bias_regularizer_ = config.lambdarank_position_bias_regularizer; + position_bias_regularization_ = config.lambdarank_position_bias_regularization; } explicit RankingObjective(const std::vector&) : seed_(0) {} @@ -122,8 +122,8 @@ class RankingObjective : public ObjectiveFunction { mutable std::vector pos_biases_; /*! \brief Learning rate to update position bias factors */ double learning_rate_; - /*! \brief Position bias regularizer */ - double position_bias_regularizer_; + /*! \brief Position bias regularization */ + double position_bias_regularization_; }; /*! @@ -323,8 +323,8 @@ class LambdarankNDCG : public RankingObjective { instance_count += instance_counts[offset]; } // L2 regularization on position bias factors - bias_first_derivative -= pos_biases_[i] * position_bias_regularizer_ * instance_count; - bias_second_derivative -= position_bias_regularizer_ * instance_count; + bias_first_derivative -= pos_biases_[i] * position_bias_regularization_ * instance_count; + bias_second_derivative -= position_bias_regularization_ * instance_count; // do Newton-Raphson step to update position bias factors pos_biases_[i] += learning_rate_ * bias_first_derivative / (std::abs(bias_second_derivative) + 0.001); } From 893405bacc372336f89f0f8e6f106ff97afe339c Mon Sep 17 00:00:00 2001 From: Pavel Metrikov <46672636+metpavel@users.noreply.github.com> Date: Sat, 26 Aug 2023 02:03:30 -0700 Subject: [PATCH 94/97] Update config.h --- include/LightGBM/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/LightGBM/config.h b/include/LightGBM/config.h index b96843b3fcc7..343abf51e17f 100644 --- a/include/LightGBM/config.h +++ b/include/LightGBM/config.h @@ -967,7 +967,7 @@ struct Config { // check = >=0.0 // desc = used only in ``lambdarank`` application when positional information is provided and position bias is modeled. Larger values reduce the inferred position bias factors. - double lambdarank_position_bias_regularizer = 0.0; + double lambdarank_position_bias_regularization = 0.0; #ifndef __NVCC__ #pragma endregion From f758b263ea30f0485ac8b1c1fc0a9c451ab8e02a Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Mon, 4 Sep 2023 02:25:17 +0000 Subject: [PATCH 95/97] update config_auto.cppp --- src/io/config_auto.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp index 746ddafa46b9..8182c9b52b93 100644 --- a/src/io/config_auto.cpp +++ b/src/io/config_auto.cpp @@ -304,7 +304,7 @@ const std::unordered_set& Config::parameter_set() { "lambdarank_truncation_level", "lambdarank_norm", "label_gain", - "lambdarank_position_bias_regularizer", + "lambdarank_position_bias_regularization", "metric", "metric_freq", "is_provide_training_metric", @@ -620,8 +620,8 @@ void Config::GetMembersFromString(const std::unordered_map(tmp_str, ','); } - GetDouble(params, "lambdarank_position_bias_regularizer", &lambdarank_position_bias_regularizer); - CHECK_GE(lambdarank_position_bias_regularizer, 0.0); + GetDouble(params, "lambdarank_position_bias_regularization", &lambdarank_position_bias_regularization); + CHECK_GE(lambdarank_position_bias_regularization, 0.0); GetInt(params, "metric_freq", &metric_freq); CHECK_GT(metric_freq, 0); @@ -758,7 +758,7 @@ std::string Config::SaveMembersToString() const { str_buf << "[lambdarank_truncation_level: " << lambdarank_truncation_level << "]\n"; str_buf << "[lambdarank_norm: " << lambdarank_norm << "]\n"; str_buf << "[label_gain: " << Common::Join(label_gain, ",") << "]\n"; - str_buf << "[lambdarank_position_bias_regularizer: " << lambdarank_position_bias_regularizer << "]\n"; + str_buf << "[lambdarank_position_bias_regularization: " << lambdarank_position_bias_regularization << "]\n"; str_buf << "[eval_at: " << Common::Join(eval_at, ",") << "]\n"; str_buf << "[multi_error_top_k: " << multi_error_top_k << "]\n"; str_buf << "[auc_mu_weights: " << Common::Join(auc_mu_weights, ",") << "]\n"; @@ -898,7 +898,7 @@ const std::unordered_map>& Config::paramet {"lambdarank_truncation_level", {}}, {"lambdarank_norm", {}}, {"label_gain", {}}, - {"lambdarank_position_bias_regularizer", {}}, + {"lambdarank_position_bias_regularization", {}}, {"metric", {"metrics", "metric_types"}}, {"metric_freq", {"output_freq"}}, {"is_provide_training_metric", {"training_metric", "is_training_metric", "train_metric"}}, @@ -1041,7 +1041,7 @@ const std::unordered_map& Config::ParameterTypes() { {"lambdarank_truncation_level", "int"}, {"lambdarank_norm", "bool"}, {"label_gain", "vector"}, - {"lambdarank_position_bias_regularizer", "double"}, + {"lambdarank_position_bias_regularization", "double"}, {"metric", "vector"}, {"metric_freq", "int"}, {"is_provide_training_metric", "bool"}, From 960c758b72eefb964bd8712b00c01107f25b780d Mon Sep 17 00:00:00 2001 From: shiyu1994 Date: Mon, 4 Sep 2023 10:32:53 +0800 Subject: [PATCH 96/97] Update docs/Advanced-Topics.rst Co-authored-by: James Lamb --- docs/Advanced-Topics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Advanced-Topics.rst b/docs/Advanced-Topics.rst index 5502e7bac904..345a1361bfa9 100644 --- a/docs/Advanced-Topics.rst +++ b/docs/Advanced-Topics.rst @@ -102,7 +102,7 @@ LightGBM could be told to account for the position bias from results being "abov ... Where ``0 = "above the fold"`` and ``1 = "requires scrolling"``. -The specific value are not important, as long as they are consistent across all observations in the training data. +The specific values are not important, as long as they are consistent across all observations in the training data. An encoding like ``100 = "above the fold"`` and ``17 = "requires scrolling"`` would result in exactly the same trained model. In that way, ``positions`` in LightGBM's API are similar to a categorical feature. From 3d934e61b15b2c72621d5c310cc4e6b17b06b471 Mon Sep 17 00:00:00 2001 From: Yu Shi Date: Mon, 4 Sep 2023 06:13:32 +0000 Subject: [PATCH 97/97] fix randomness in test case for gpu --- tests/python_package_test/test_engine.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index d4d59f617919..25413d7ea072 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -747,6 +747,7 @@ def test_ranking_prediction_early_stopping(): with pytest.raises(AssertionError): np.testing.assert_allclose(ret_early, ret_early_more_strict) + # Simulates position bias for a given ranking dataset. # The ouput dataset is identical to the input one with the exception for the relevance labels. # The new labels are generated according to an instance of a cascade user model: @@ -808,7 +809,8 @@ def get_pclick(label): positions_all.extend(positions) f_dataset_out.close() return positions_all - + + @pytest.mark.skipif(getenv('TASK', '') == 'cuda', reason='Positions in learning to rank is not supported in CUDA version yet') def test_ranking_with_position_information_with_file(tmp_path): rank_example_dir = Path(__file__).absolute().parents[2] / 'examples' / 'lambdarank' @@ -822,7 +824,7 @@ def test_ranking_with_position_information_with_file(tmp_path): 'min_data_in_leaf': 50, 'min_sum_hessian_in_leaf': 5.0 } - + # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory positions = simulate_position_bias(str(rank_example_dir / 'rank.train'), str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train'), baseline_feature=34) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query')) @@ -866,9 +868,12 @@ def test_ranking_with_position_information_with_dataset_constructor(tmp_path): 'bagging_freq': 1, 'bagging_fraction': 0.9, 'min_data_in_leaf': 50, - 'min_sum_hessian_in_leaf': 5.0 + 'min_sum_hessian_in_leaf': 5.0, + 'num_threads': 1, + 'deterministic': True, + 'seed': 0 } - + # simulate position bias for the train dataset and put the train dataset with biased labels to temp directory positions = simulate_position_bias(str(rank_example_dir / 'rank.train'), str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train'), baseline_feature=34) copyfile(str(rank_example_dir / 'rank.train.query'), str(tmp_path / 'rank.train.query'))