clear memory of sample data right after BinMapper is constructed to s…

…ave memory (#4890) Sample data is useless after BinMapper is constructed, but the corresponding memory is still there before feature extraction is finished.
microsoft · Dec 23, 2021 · 2ef3cb8 · 2ef3cb8
1 parent e4c0ca5
commit 2ef3cb8
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/src/io/dataset_loader.cpp b/src/io/dataset_loader.cpp
@@ -234,8 +234,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
       auto sample_data = SampleTextDataFromMemory(text_data);
       CheckSampleSize(sample_data.size(),
                       static_cast<size_t>(dataset->num_data_));
-      // construct feature bin mappers
+      // construct feature bin mappers & clear sample data
       ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
+      std::vector<std::string>().swap(sample_data);
       if (dataset->has_raw()) {
         dataset->ResizeRaw(dataset->num_data_);
       }
@@ -254,8 +255,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
       }
       CheckSampleSize(sample_data.size(),
                       static_cast<size_t>(dataset->num_data_));
-      // construct feature bin mappers
+      // construct feature bin mappers & clear sample data
       ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
+      std::vector<std::string>().swap(sample_data);
       if (dataset->has_raw()) {
         dataset->ResizeRaw(dataset->num_data_);
       }