add support for generating memory profile

Dockerfile

@@ -70,6 +70,7 @@ COPY --from=bin-builder /quickwit/config/quickwit.yaml /quickwit/config/quickwit
 ENV QW_CONFIG=/quickwit/config/quickwit.yaml
 ENV QW_DATA_DIR=/quickwit/qwdata
 ENV QW_LISTEN_ADDRESS=0.0.0.0
+ENV MALLOC_CONF=prof:true
 
 RUN quickwit --version
 

quickwit/Cargo.toml

@@ -97,6 +97,7 @@ chrono = { version = "0.4", default-features = false, features = [
 clap = { version = "4.5.0", features = ["env", "string"] }
 coarsetime = "0.1.33"
 colored = "2.1.0"
+common-mem-prof = { git = "https://github.com/GreptimeTeam/greptimedb", rev = "fcff66e03904d80aacb91b8edd4e15240161d264" }
 console-subscriber = "0.1.8"
 criterion = { version = "0.5", features = ["async_tokio"] }
 cron = "0.12.0"

quickwit/quickwit-serve/Cargo.toml

@@ -16,6 +16,7 @@ async-trait = { workspace = true }
 base64 = { workspace = true }
 bytes = { workspace = true }
 bytesize = { workspace = true }
+common-mem-prof = { workspace = true }
 elasticsearch-dsl = "0.4.15"
 flate2 = { workspace = true }
 futures = { workspace = true }

quickwit/quickwit-serve/src/developer_api/mod.rs

@@ -20,6 +20,7 @@
 mod debug;
 mod log_level;
 
+mod mprof;
 #[cfg_attr(not(feature = "pprof"), path = "pprof_disabled.rs")]
 mod pprof;
 
@@ -28,6 +29,7 @@ mod server;
 
 use debug::debug_handler;
 use log_level::log_level_handler;
+use mprof::mprof_handler;
 use pprof::pprof_handlers;
 use quickwit_cluster::Cluster;
 use quickwit_proto::control_plane::ControlPlaneServiceClient;
@@ -52,6 +54,7 @@ pub(crate) fn developer_api_routes(
             debug_handler(cluster.clone())
                 .or(log_level_handler(env_filter_reload_fn.clone()))
                 .or(pprof_handlers())
+                .or(mprof_handler())
                 .or(rebuild_plan_handler(control_plane_client)),
         )
         .recover(recover_fn)

quickwit/quickwit-serve/src/developer_api/mprof.rs

@@ -0,0 +1,37 @@
+// Copyright (C) 2024 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at [email protected].
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use warp::{Filter, Rejection};
+
+pub(crate) fn mprof_handler(
+) -> impl Filter<Extract = (impl warp::Reply,), Error = Rejection> + Clone {
+    warp::path!("mprof" / "flamegraph").then(get_memory_map)
+}
+
+#[cfg(feature = "pprof")]
+async fn get_memory_map() -> impl warp::Reply {
+    common_mem_prof::dump_profile()
+        .await
+        .unwrap_or_else(|e| e.to_string().into_bytes())
+}
+
+#[cfg(not(feature = "pprof"))]
+async fn get_memory_map() -> impl warp::Reply {
+    "mprof disabled"
+}