Skip to content

Commit

Permalink
chore: add prometheus-process feature, use gauge
Browse files Browse the repository at this point in the history
  • Loading branch information
@captainlee1024
captainlee1024 committed Jul 11, 2024
1 parent 48c7afd commit a6bf9cf
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 117 deletions.
2 changes: 1 addition & 1 deletion metrics/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
hyper = { version = "0.14.16", default-features = false, features = ["http1", "server", "tcp"] }
log = "0.4.0"
prometheus = { version = "0.13.0", default-features = false }
prometheus = { version = "0.13.4", features = ["process", "libc", "procfs"] }
thiserror = "1.0"
tokio = { version = "1.21.0", features = ["parking_lot"] }
env_logger = "0.10"
Expand Down
102 changes: 45 additions & 57 deletions metrics/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ pub mod batch_processing_time_histogram_labels {
pub const LABELS_BLOCK_NUMBER: &str = "block_number";
pub const LABELS_CHUNK_ID: &str = "chunk_id";
pub const LABELS_STEP: &str = "step";
pub const LABELS_STEP_BATCH: &str = "step_batch";
pub const LABELS_STEP_BATCH_STARK: &str = "step_batch_stark";
pub const LABELS_STEP_BATCH_C12_STARK: &str = "step_batch_c12_stark";
pub const LABELS_STEP_AGG: &str = "step_agg";
pub const LABELS_STEP_FINAL: &str = "step_final";
pub const LABELS_FUNCTION: &str = "function";
Expand All @@ -48,15 +49,27 @@ pub enum BatchProcessingTimeHistogramLabels {
}

pub enum Step {
Batch,
Batch(Batch),
Agg,
Final,
}

pub enum Batch {
BatchStark,
C12Stark,
}

impl From<Step> for String {
fn from(value: Step) -> Self {
match value {
Step::Batch => batch_processing_time_histogram_labels::LABELS_STEP_BATCH.to_string(),
Step::Batch(value) => match value {
Batch::BatchStark => {
batch_processing_time_histogram_labels::LABELS_STEP_BATCH_STARK.to_string()
}
Batch::C12Stark => {
batch_processing_time_histogram_labels::LABELS_STEP_BATCH_STARK.to_string()
}
},
Step::Agg => batch_processing_time_histogram_labels::LABELS_STEP_AGG.to_string(),
Step::Final => batch_processing_time_histogram_labels::LABELS_STEP_FINAL.to_string(),
}
Expand Down Expand Up @@ -101,8 +114,6 @@ impl PrometheusMetrics {
.namespace("eigen_prover")
.const_label("type", "gauge"),
&[
batch_processing_time_gauge_labels::LABELS_BLOCK_NUMBER,
batch_processing_time_gauge_labels::LABELS_CHUNK_ID,
batch_processing_time_gauge_labels::LABELS_STEP,
batch_processing_time_gauge_labels::LABELS_FUNCTION,
],
Expand All @@ -117,8 +128,6 @@ impl PrometheusMetrics {
.namespace("eigen_prover")
.const_label("type", "histogram"),
&[
batch_processing_time_histogram_labels::LABELS_BLOCK_NUMBER,
batch_processing_time_histogram_labels::LABELS_CHUNK_ID,
batch_processing_time_histogram_labels::LABELS_STEP,
batch_processing_time_histogram_labels::LABELS_FUNCTION,
],
Expand All @@ -138,43 +147,24 @@ impl PrometheusMetrics {
})
}

pub fn observe_prover_processing_time_gauge(
&self,
block_number: String,
chunk_id: String,
step: Step,
function: Function,
value: f64,
) {
pub fn observe_prover_processing_time_gauge(&self, step: Step, function: Function, value: f64) {
let step_string: String = step.into();
let function_string: String = function.into();
self.prover_processing_time_gauge
.with_label_values(&[
&block_number,
&chunk_id,
step_string.as_str(),
function_string.as_str(),
])
.with_label_values(&[step_string.as_str(), function_string.as_str()])
.set(value);
}

pub fn observe_prover_processing_time_histogram(
pub fn _observe_prover_processing_time_histogram(
&self,
block_number: String,
chunk_id: String,
step: Step,
function: Function,
value: f64,
) {
let step_string: String = step.into();
let function_string: String = function.into();
self.prover_processing_time_histogram
.with_label_values(&[
&block_number,
&chunk_id,
step_string.as_str(),
function_string.as_str(),
])
.with_label_values(&[step_string.as_str(), function_string.as_str()])
.observe(value);
}
}
Expand Down Expand Up @@ -256,25 +246,21 @@ mod tests {
.lock()
.unwrap()
.observe_prover_processing_time_gauge(
"0000000001".to_string(),
"1".to_string(),
Step::Batch,
Step::Batch(Batch::BatchStark),
Function::Setup,
1.0,
);
PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0000000001".to_string(),
"1".to_string(),
Step::Batch,
._observe_prover_processing_time_histogram(
Step::Batch(Batch::BatchStark),
Function::Setup,
1.0,
);

let prometheus_port =
std::env::var("PROMETHEUS_ADDR").unwrap_or("0.0.0.0:33032".to_string());
std::env::var("PROMETHEUS_ADDR").unwrap_or("0.0.0.0:43032".to_string());
let prometheus_addr: SocketAddr = prometheus_port.parse().expect("Invalid socket address");
// let prometheus_addr = ([127, 0, 0, 1], 33032).into();
tokio::spawn(async move {
Expand All @@ -300,30 +286,32 @@ mod tests {
let body_content = str::from_utf8(&body_bytes).unwrap();

println!("Response body: {}", body_content);
// # HELP eigen_prover_prover_processing_time_gauge Prover processing time in seconds
// Response body: # HELP eigen_prover_prover_processing_time_gauge Prover processing time in seconds
// # TYPE eigen_prover_prover_processing_time_gauge gauge
// eigen_prover_prover_processing_time_gauge{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="gauge"} 1
// eigen_prover_prover_processing_time_gauge{function="function_setup",step="step_batch_stark",type="gauge"} 1
// # HELP eigen_prover_prover_processing_time_histogram Prover processing time in seconds
// # TYPE eigen_prover_prover_processing_time_histogram histogram
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.005"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.01"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.025"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.05"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.1"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.25"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="0.5"} 0
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="1"} 1
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="2.5"} 1
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="5"} 1
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="10"} 1
// eigen_prover_prover_processing_time_histogram_bucket{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram",le="+Inf"} 1
// eigen_prover_prover_processing_time_histogram_sum{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram"} 1
// eigen_prover_prover_processing_time_histogram_count{block_number="0000000001",chunk_id="1",function="function_setup",step="step_batch",type="histogram"} 1

let expect_gauge = "eigen_prover_prover_processing_time_gauge{block_number=\"0000000001\",chunk_id=\"1\",function=\"function_setup\",step=\"step_batch\",type=\"gauge\"} 1";
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.005"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.01"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.025"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.05"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.1"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.25"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="0.5"} 0
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="1"} 1
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="2.5"} 1
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="5"} 1
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="10"} 1
// eigen_prover_prover_processing_time_histogram_bucket{function="function_setup",step="step_batch_stark",type="histogram",le="+Inf"} 1
// eigen_prover_prover_processing_time_histogram_sum{function="function_setup",step="step_batch_stark",type="histogram"} 1
// eigen_prover_prover_processing_time_histogram_count{function="function_setup",step="step_batch_stark",type="histogram"} 1

// eigen_prover_prover_processing_time_gauge{function="function_setup",step="step_batch_stark",type="gauge"} 1
let expect_gauge = "eigen_prover_prover_processing_time_gauge{function=\"function_setup\",step=\"step_batch_stark\",type=\"gauge\"} 1";
assert!(body_content.contains(expect_gauge));

let expect_histogram_sum = "eigen_prover_prover_processing_time_histogram_sum{block_number=\"0000000001\",chunk_id=\"1\",function=\"function_setup\",step=\"step_batch\",type=\"histogram\"} 1";
// eigen_prover_prover_processing_time_histogram_sum{function="function_setup",step="step_batch_stark",type="histogram"} 1
let expect_histogram_sum = "eigen_prover_prover_processing_time_histogram_sum{function=\"function_setup\",step=\"step_batch_stark\",type=\"histogram\"} 1";
assert!(body_content.contains(expect_histogram_sum));
}
}
24 changes: 6 additions & 18 deletions prover/src/provers/agg_prover.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::Setup,
setup_elapsed.as_secs_f64(),
Expand All @@ -161,9 +159,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::Exec,
exec_elapsed.as_secs_f64(),
Expand Down Expand Up @@ -194,9 +190,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::StarkProve,
stark_prove_elapsed.as_secs_f64(),
Expand Down Expand Up @@ -229,9 +223,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::Exec,
exec_elapsed.as_secs_f64(),
Expand All @@ -254,9 +246,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::StarkProve,
stark_prove_elapsed.as_secs_f64(),
Expand All @@ -269,9 +259,7 @@ impl Prover<AggContext> for AggProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
"0".to_string(),
"0".to_string(),
.observe_prover_processing_time_gauge(
Step::Agg,
Function::Total,
prove_elapsed.as_secs_f64(),
Expand Down
46 changes: 17 additions & 29 deletions prover/src/provers/batch_prover.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ use anyhow::Result;
use powdr::number::{FieldElement, GoldilocksField};

use dsl_compile::circom_compiler;
use metrics::{Function, Step};
use metrics::Batch::BatchStark;
use metrics::{Batch, Function, Step};
use recursion::{compressor12_exec::exec, compressor12_setup::setup};
use starky::prove::stark_prove;
use std::{fs, io::Read};
Expand Down Expand Up @@ -108,10 +109,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(Batch::BatchStark),
Function::Setup,
setup_elapsed.as_secs_f64(),
);
Expand All @@ -129,10 +128,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(BatchStark),
Function::Exec,
exec_elapsed.as_secs_f64(),
);
Expand All @@ -156,10 +153,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(BatchStark),
Function::StarkProve,
stark_prove_elapsed.as_secs_f64(),
);
Expand Down Expand Up @@ -189,10 +184,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(Batch::C12Stark),
Function::Setup,
c12_setup_elapsed.as_secs_f64(),
);
Expand All @@ -207,13 +200,12 @@ impl Prover<BatchContext> for BatchProver {
&c12_stark.commit_file,
)?;
let c12_exec_elapsed = c12_exec_start.elapsed();
log::info!("c12 proof: compress exec elapsed: {:?}", c12_exec_elapsed);
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(Batch::C12Stark),
Function::Exec,
c12_exec_elapsed.as_secs_f64(),
);
Expand All @@ -236,10 +228,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(Batch::C12Stark),
Function::StarkProve,
c12_stark_prove_elapsed.as_secs_f64(),
);
Expand All @@ -249,10 +239,8 @@ impl Prover<BatchContext> for BatchProver {
metrics::PROMETHEUS_METRICS
.lock()
.unwrap()
.observe_prover_processing_time_histogram(
ctx.task_id.clone(),
ctx.chunk_id.clone(),
Step::Batch,
.observe_prover_processing_time_gauge(
Step::Batch(BatchStark),
Function::Total,
prove_elapsed.as_secs_f64(),
);
Expand Down
Loading

0 comments on commit a6bf9cf

Please sign in to comment.