[c++] Initial Work for Pairwise Ranking

CMakeLists.txt

@@ -439,6 +439,8 @@ set(
       src/io/parser.cpp
       src/io/train_share_states.cpp
       src/io/tree.cpp
+      src/io/pairwise_lambdarank_bin.cpp
+      src/io/pairwise_ranking_feature_group.cpp
       src/metric/dcg_calculator.cpp
       src/metric/metric.cpp
       src/network/linker_topo.cpp

docs/Parameters.rst

@@ -168,6 +168,8 @@ Core Parameters
 
       -  ``rank_xendcg`` is faster than and achieves the similar performance as ``lambdarank``
 
+      -  ``pairwise_lambdarank``, pairwise lambdarank algorithm
+
       -  label should be ``int`` type, and larger number represents the higher relevance (e.g. 0:bad, 1:fair, 2:good, 3:perfect)
 
    -  custom objective function (gradients and hessians not computed directly by LightGBM)
@@ -413,6 +415,10 @@ Learning Control Parameters
 
    -  random seed for bagging
 
+-  ``bagging_by_query`` :raw-html:`<a id="bagging_by_query" title="Permalink to this parameter" href="#bagging_by_query">&#x1F517;&#xFE0E;</a>`, default = ``false``, type = bool
+
+   -  whether to do bagging sample by query
+
 -  ``feature_fraction`` :raw-html:`<a id="feature_fraction" title="Permalink to this parameter" href="#feature_fraction">&#x1F517;&#xFE0E;</a>`, default = ``1.0``, type = double, aliases: ``sub_feature``, ``colsample_bytree``, constraints: ``0.0 < feature_fraction <= 1.0``
 
    -  LightGBM will randomly select a subset of features on each iteration (tree) if ``feature_fraction`` is smaller than ``1.0``. For example, if you set it to ``0.8``, LightGBM will select 80% of features before training each tree
@@ -1214,6 +1220,66 @@ Objective Parameters
 
    -  *New in version 4.1.0*
 
+-  ``use_differential_feature_in_pairwise_ranking`` :raw-html:`<a id="use_differential_feature_in_pairwise_ranking" title="Permalink to this parameter" href="#use_differential_feature_in_pairwise_ranking">&#x1F517;&#xFE0E;</a>`, default = ``false``, type = bool
+
+   -  whether to use differential features in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_model_indirect_comparison`` :raw-html:`<a id="pairwise_lambdarank_model_indirect_comparison" title="Permalink to this parameter" href="#pairwise_lambdarank_model_indirect_comparison">&#x1F517;&#xFE0E;</a>`, default = ``false``, type = bool
+
+   -  whether to additionaly perform indirect document comparison in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_model_conditional_rel`` :raw-html:`<a id="pairwise_lambdarank_model_conditional_rel" title="Permalink to this parameter" href="#pairwise_lambdarank_model_conditional_rel">&#x1F517;&#xFE0E;</a>`, default = ``false``, type = bool
+
+   -  whether to model conditional document relevance (given documents ranked above) in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_indirect_comparison_above_only`` :raw-html:`<a id="pairwise_lambdarank_indirect_comparison_above_only" title="Permalink to this parameter" href="#pairwise_lambdarank_indirect_comparison_above_only">&#x1F517;&#xFE0E;</a>`, default = ``true``, type = bool
+
+   -  whether to limit the indirect document comparison to only auxilliary documents ranked above in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_logarithmic_discounts`` :raw-html:`<a id="pairwise_lambdarank_logarithmic_discounts" title="Permalink to this parameter" href="#pairwise_lambdarank_logarithmic_discounts">&#x1F517;&#xFE0E;</a>`, default = ``true``, type = bool
+
+   -  whether to use logarithmic discounts when converting pairwise scores into pointwise in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_hard_pairwise_preference`` :raw-html:`<a id="pairwise_lambdarank_hard_pairwise_preference" title="Permalink to this parameter" href="#pairwise_lambdarank_hard_pairwise_preference">&#x1F517;&#xFE0E;</a>`, default = ``false``, type = bool
+
+   -  whether to use hard pairwise preference when converting pairwise scores into pointwise in pairwise ranking
+
+   -  used only in ``pairwise_lambdarank`` application
+
+-  ``pairwise_lambdarank_train_pairing_approach`` :raw-html:`<a id="pairwise_lambdarank_train_pairing_approach" title="Permalink to this parameter" href="#pairwise_lambdarank_train_pairing_approach">&#x1F517;&#xFE0E;</a>`, default = ``std::string("different_relevance")``, type = string
+
+   -  pairing appraoch for training dataset
+
+   -  used only in ``pairwise_lambdarank`` application
+
+   -  with ``different_relevance``, only consider pairs with difference relevance score
+
+   -  with ``at_least_one_relevant``, only consider pairs with at least one relevant item
+
+   -  with ``all``, all pairs will be used
+
+-  ``pairwise_lambdarank_valid_pairing_approach`` :raw-html:`<a id="pairwise_lambdarank_valid_pairing_approach" title="Permalink to this parameter" href="#pairwise_lambdarank_valid_pairing_approach">&#x1F517;&#xFE0E;</a>`, default = ``std::string("different_relevance")``, type = string
+
+   -  pairing appraoch for validation dataset
+
+   -  used only in ``pairwise_lambdarank`` application
+
+   -  with ``different_relevance``, only consider pairs with difference relevance score
+
+   -  with ``at_least_one_relevant``, only consider pairs with at least one relevant item
+
+   -  with ``all``, all pairs will be used
+
 Metric Parameters
 -----------------
 

include/LightGBM/bin.h

@@ -14,6 +14,7 @@
 #include <functional>
 #include <sstream>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 namespace LightGBM {
@@ -305,6 +306,10 @@ class Bin {
   */
   virtual BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin, uint32_t most_freq_bin) const = 0;
 
+  virtual BinIterator* GetUnpairedIterator(uint32_t /* min_bin */, uint32_t /* max_bin */, uint32_t /* most_freq_bin */) const {
+    return nullptr;
+  }
+
   /*!
   * \brief Save binary data to file
   * \param file File want to write
@@ -466,6 +471,64 @@ class Bin {
   */
   static Bin* CreateSparseBin(data_size_t num_data, int num_bin);
 
+  /*!
+  * \brief Create object for bin data of the first feature in pair, used for pairwise ranking, for an original dense bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \return The bin data object
+  */
+  static Bin* CreateDensePairwiseRankingFirstBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map);
+
+  /*!
+  * \brief Create object for bin data of the first feature in pair, used for pairwise ranking, for an original sparse bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \return The bin data object
+  */
+  static Bin* CreateSparsePairwiseRankingFirstBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map);
+
+  /*!
+  * \brief Create object for bin data of the second feature in pair, used for pairwise ranking, for an original dense bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \return The bin data object
+  */
+  static Bin* CreateDensePairwiseRankingSecondBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map);
+
+  /*!
+  * \brief Create object for bin data of the second feature in pair, used for pairwise ranking, for an original sparse bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \return The bin data object
+  */
+  static Bin* CreateSparsePairwiseRankingSecondBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map);
+
+  /*!
+  * \brief Create object for bin data of the differential feature in pair, used for pairwise ranking, for an original dense bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \param diff_bin_mappers Bin mappers for differential features in this group
+  * \param bin_offsets Bin offsets in feature group
+  * \return The bin data object
+  */
+  static Bin* CreateDensePairwiseRankingDiffBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map, const std::vector<std::unique_ptr<const BinMapper>>* diff_bin_mappers, const std::vector<std::unique_ptr<const BinMapper>>* ori_bin_mappers, const std::vector<uint32_t>* bin_offsets, const std::vector<uint32_t>* diff_bin_offsets);
+
+  /*!
+  * \brief Create object for bin data of the differential feature in pair, used for pairwise ranking, for an original sparse bin
+  * \param num_data Size of the pairwise dataset
+  * \param num_bin Number of bin
+  * \param paired_ranking_item_index_map Map from data index to the original index for items in the pair
+  * \param diff_bin_mappers Bin mappers for differential features in this group
+  * \param bin_offsets Bin offsets in feature group
+  * \return The bin data object
+  */
+  static Bin* CreateSparsePairwiseRankingDiffBin(data_size_t num_original_data, int num_bin, data_size_t num_pairs, const std::pair<data_size_t, data_size_t>* paired_ranking_item_index_map, const std::vector<std::unique_ptr<const BinMapper>>* diff_bin_mappers, const std::vector<std::unique_ptr<const BinMapper>>* ori_bin_mappers, const std::vector<uint32_t>* bin_offsets, const std::vector<uint32_t>* diff_bin_offsets);
+
   /*!
   * \brief Deep copy the bin
   */
@@ -474,6 +537,8 @@ class Bin {
   virtual const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, std::vector<BinIterator*>* bin_iterator, const int num_threads) const = 0;
 
   virtual const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, BinIterator** bin_iterator) const = 0;
+
+  int group_index_ = -1;
 };
 
 
@@ -495,6 +560,8 @@ class MultiValBin {
                           const data_size_t* used_indices,
                           data_size_t num_used_indices) = 0;
 
+  virtual void DumpContent() const {}
+
   virtual MultiValBin* CreateLike(data_size_t num_data, int num_bin,
                                   int num_feature,
                                   double estimate_element_per_row,
@@ -588,12 +655,14 @@ class MultiValBin {
   virtual bool IsSparse() = 0;
 
   static MultiValBin* CreateMultiValBin(data_size_t num_data, int num_bin,
-                                        int num_feature, double sparse_rate, const std::vector<uint32_t>& offsets);
+                                        int num_feature, double sparse_rate, const std::vector<uint32_t>& offsets, const bool use_pairwise_ranking,
+                                        const std::pair<data_size_t, data_size_t>* paired_ranking_item_global_index_map);
 
   static MultiValBin* CreateMultiValDenseBin(data_size_t num_data, int num_bin,
-                                             int num_feature, const std::vector<uint32_t>& offsets);
+                                             int num_feature, const std::vector<uint32_t>& offsets, const bool use_pairwise_ranking,
+                                             const std::pair<data_size_t, data_size_t>* paired_ranking_item_global_index_map);
 
-  static MultiValBin* CreateMultiValSparseBin(data_size_t num_data, int num_bin, double estimate_element_per_row);
+  static MultiValBin* CreateMultiValSparseBin(data_size_t num_data, int num_bin, double estimate_element_per_row, const bool use_pairwise_ranking, const std::pair<data_size_t, data_size_t>* paired_ranking_item_global_index_map);
 
   static constexpr double multi_val_bin_sparse_threshold = 0.25f;
 

include/LightGBM/config.h

@@ -36,6 +36,11 @@ enum TaskType {
 };
 const int kDefaultNumLeaves = 31;
 
+/*! \brief Types of pairwise ranking mode */
+enum PairwiseRankingMode {
+  kNone, kFull, kRelevance, kManual
+};
+
 struct Config {
  public:
   Config() {}
@@ -157,6 +162,7 @@ struct Config {
   // descl2 = ``lambdarank``, `lambdarank <https://proceedings.neurips.cc/paper_files/paper/2006/file/af44c4c56f385c43f2529f9b1b018f6a-Paper.pdf>`__ objective. `label_gain <#label_gain>`__ can be used to set the gain (weight) of ``int`` label and all values in ``label`` must be smaller than number of elements in ``label_gain``
   // descl2 = ``rank_xendcg``, `XE_NDCG_MART <https://arxiv.org/abs/1911.09798>`__ ranking objective function, aliases: ``xendcg``, ``xe_ndcg``, ``xe_ndcg_mart``, ``xendcg_mart``
   // descl2 = ``rank_xendcg`` is faster than and achieves the similar performance as ``lambdarank``
+  // descl2 = ``pairwise_lambdarank``, pairwise lambdarank algorithm
   // descl2 = label should be ``int`` type, and larger number represents the higher relevance (e.g. 0:bad, 1:fair, 2:good, 3:perfect)
   // desc = custom objective function (gradients and hessians not computed directly by LightGBM)
   // descl2 = ``custom``
@@ -358,6 +364,9 @@ struct Config {
   // desc = random seed for bagging
   int bagging_seed = 3;
 
+  // desc = whether to do bagging sample by query
+  bool bagging_by_query = false;
+
   // alias = sub_feature, colsample_bytree
   // check = >0.0
   // check = <=1.0
@@ -995,6 +1004,44 @@ struct Config {
   // desc = *New in version 4.1.0*
   double lambdarank_position_bias_regularization = 0.0;
 
+  // desc = whether to use differential features in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool use_differential_feature_in_pairwise_ranking = false;
+
+  // desc = whether to additionaly perform indirect document comparison in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool pairwise_lambdarank_model_indirect_comparison = false;
+
+  // desc = whether to model conditional document relevance (given documents ranked above) in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool pairwise_lambdarank_model_conditional_rel = false;
+
+  // desc = whether to limit the indirect document comparison to only auxilliary documents ranked above in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool pairwise_lambdarank_indirect_comparison_above_only = true;
+
+  // desc = whether to use logarithmic discounts when converting pairwise scores into pointwise in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool pairwise_lambdarank_logarithmic_discounts = true;
+
+  // desc = whether to use hard pairwise preference when converting pairwise scores into pointwise in pairwise ranking
+  // desc = used only in ``pairwise_lambdarank`` application
+  bool pairwise_lambdarank_hard_pairwise_preference = false;
+
+  // desc = pairing appraoch for training dataset
+  // desc = used only in ``pairwise_lambdarank`` application
+  // desc = with ``different_relevance``, only consider pairs with difference relevance score
+  // desc = with ``at_least_one_relevant``, only consider pairs with at least one relevant item
+  // desc = with ``all``, all pairs will be used
+  std::string pairwise_lambdarank_train_pairing_approach = std::string("different_relevance");
+
+  // desc = pairing appraoch for validation dataset
+  // desc = used only in ``pairwise_lambdarank`` application
+  // desc = with ``different_relevance``, only consider pairs with difference relevance score
+  // desc = with ``at_least_one_relevant``, only consider pairs with at least one relevant item
+  // desc = with ``all``, all pairs will be used
+  std::string pairwise_lambdarank_valid_pairing_approach = std::string("different_relevance");
+
   #ifndef __NVCC__
   #pragma endregion
 

include/LightGBM/cuda/cuda_objective_function.hpp

@@ -49,6 +49,11 @@ class CUDAObjectiveInterface: public HOST_OBJECTIVE {
     SynchronizeCUDADevice(__FILE__, __LINE__);
   }
 
+  void GetGradients(const double* scores, const data_size_t /*num_sampled_queries*/, const data_size_t* /*sampled_query_indices*/, score_t* gradients, score_t* hessians) const override {
+    LaunchGetGradientsKernel(scores, gradients, hessians);
+    SynchronizeCUDADevice(__FILE__, __LINE__);
+  }
+
   void RenewTreeOutputCUDA(const double* score, const data_size_t* data_indices_in_leaf, const data_size_t* num_data_in_leaf,
     const data_size_t* data_start_in_leaf, const int num_leaves, double* leaf_value) const override {
     global_timer.Start("CUDAObjectiveInterface::LaunchRenewTreeOutputCUDAKernel");