From 810d7949b04f73d16b08c445926d4c094efbb51e Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Thu, 7 Mar 2024 19:33:53 +0100 Subject: [PATCH 1/3] export queue capacity --- src/lib.rs | 9 +++++++++ src/mem/queue.rs | 9 +++++++++ src/mem/queues.rs | 17 +++++++++++++---- src/multi_record_log.rs | 4 ++-- src/tests.rs | 12 ++++++++---- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3def6f7..3a31b15 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,15 @@ impl<'a> Record<'a> { } } +/// Memory used by mrecordlog +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MemoryUsage { + /// Actual size of the memory used + pub size: usize, + /// Capacity allocated, a part of which may be unused right now + pub capacity: usize, +} + #[cfg(test)] mod tests; diff --git a/src/mem/queue.rs b/src/mem/queue.rs index 5e75e10..74b66a2 100644 --- a/src/mem/queue.rs +++ b/src/mem/queue.rs @@ -22,6 +22,10 @@ impl RollingBuffer { self.buffer.len() } + fn capacity(&self) -> usize { + self.buffer.capacity() + } + fn clear(&mut self) { self.buffer.clear(); self.buffer.shrink_to_fit(); @@ -250,4 +254,9 @@ impl MemQueue { self.concatenated_records.len() + self.record_metas.len() * std::mem::size_of::() } + + pub fn capacity(&self) -> usize { + self.concatenated_records.capacity() + + self.record_metas.capacity() * std::mem::size_of::() + } } diff --git a/src/mem/queues.rs b/src/mem/queues.rs index abf62f1..123066f 100644 --- a/src/mem/queues.rs +++ b/src/mem/queues.rs @@ -6,7 +6,7 @@ use tracing::{info, warn}; use crate::error::{AlreadyExists, AppendError, MissingQueue}; use crate::mem::MemQueue; use crate::rolling::FileNumber; -use crate::Record; +use crate::{MemoryUsage, Record}; #[derive(Default)] pub(crate) struct MemQueues { @@ -154,10 +154,19 @@ impl MemQueues { } } - pub fn size(&self) -> usize { - self.queues + pub fn size(&self) -> MemoryUsage { + let size = self + .queues .iter() .map(|(name, queue)| name.len() + queue.size()) - .sum() + .sum(); + + let capacity = self + .queues + .iter() + .map(|(name, queue)| name.capacity() + queue.capacity()) + .sum(); + + MemoryUsage { size, capacity } } } diff --git a/src/multi_record_log.rs b/src/multi_record_log.rs index 0fe3f50..35b2f69 100644 --- a/src/multi_record_log.rs +++ b/src/multi_record_log.rs @@ -13,7 +13,7 @@ use crate::mem::MemQueue; use crate::record::{MultiPlexedRecord, MultiRecord}; use crate::recordlog::RecordWriter; use crate::rolling::RollingWriter; -use crate::{mem, Record}; +use crate::{mem, MemoryUsage, Record}; pub struct MultiRecordLog { record_log_writer: crate::recordlog::RecordWriter, @@ -356,7 +356,7 @@ impl MultiRecordLog { } /// Returns the quantity of data stored in the in memory queue. - pub fn memory_usage(&self) -> usize { + pub fn memory_usage(&self) -> MemoryUsage { self.in_mem_queues.size() } diff --git a/src/tests.rs b/src/tests.rs index 8df2e75..46a3145 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -323,21 +323,25 @@ fn test_multi_record_size() { let tempdir = tempfile::tempdir().unwrap(); { let mut multi_record_log = MultiRecordLog::open(tempdir.path()).unwrap(); - assert_eq!(multi_record_log.memory_usage(), 0); + assert_eq!(multi_record_log.memory_usage().size, 0); + assert_eq!(multi_record_log.memory_usage().capacity, 0); multi_record_log.create_queue("queue").unwrap(); let size_mem_create = multi_record_log.memory_usage(); - assert!(size_mem_create > 0); + assert!(size_mem_create.size > 0); + assert!(size_mem_create.capacity >= size_mem_create.size); multi_record_log .append_record("queue", None, &b"hello"[..]) .unwrap(); let size_mem_append = multi_record_log.memory_usage(); - assert!(size_mem_append > size_mem_create); + assert!(size_mem_append.size > size_mem_create.size); + assert!(size_mem_append.capacity >= size_mem_append.size); + assert!(size_mem_append.capacity >= size_mem_create.capacity); multi_record_log.truncate("queue", 0).unwrap(); let size_mem_truncate = multi_record_log.memory_usage(); - assert!(size_mem_truncate < size_mem_append); + assert!(size_mem_truncate.size < size_mem_append.size); } } From 44389f99dc7318beb6b0dba90069c9f44cd7c643 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Thu, 7 Mar 2024 19:50:43 +0100 Subject: [PATCH 2/3] include disk usage with mem usage --- src/lib.rs | 10 ++++++---- src/mem/queues.rs | 7 ++++--- src/multi_record_log.rs | 21 ++++++++------------- src/tests.rs | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3a31b15..2dd5132 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,13 +28,15 @@ impl<'a> Record<'a> { } } -/// Memory used by mrecordlog +/// Ressource used by mrecordlog #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MemoryUsage { +pub struct RessourceUsage { /// Actual size of the memory used - pub size: usize, + pub memory_size: usize, /// Capacity allocated, a part of which may be unused right now - pub capacity: usize, + pub memory_capacity: usize, + /// Disk size used + pub disk_size: usize, } #[cfg(test)] diff --git a/src/mem/queues.rs b/src/mem/queues.rs index 123066f..ebd11c8 100644 --- a/src/mem/queues.rs +++ b/src/mem/queues.rs @@ -6,7 +6,7 @@ use tracing::{info, warn}; use crate::error::{AlreadyExists, AppendError, MissingQueue}; use crate::mem::MemQueue; use crate::rolling::FileNumber; -use crate::{MemoryUsage, Record}; +use crate::{Record}; #[derive(Default)] pub(crate) struct MemQueues { @@ -154,7 +154,8 @@ impl MemQueues { } } - pub fn size(&self) -> MemoryUsage { + /// Return a tuple of (size, capacity) of memory used by the memqueues + pub fn size(&self) -> (usize, usize) { let size = self .queues .iter() @@ -167,6 +168,6 @@ impl MemQueues { .map(|(name, queue)| name.capacity() + queue.capacity()) .sum(); - MemoryUsage { size, capacity } + (size, capacity) } } diff --git a/src/multi_record_log.rs b/src/multi_record_log.rs index 35b2f69..8ed38d7 100644 --- a/src/multi_record_log.rs +++ b/src/multi_record_log.rs @@ -13,7 +13,7 @@ use crate::mem::MemQueue; use crate::record::{MultiPlexedRecord, MultiRecord}; use crate::recordlog::RecordWriter; use crate::rolling::RollingWriter; -use crate::{mem, MemoryUsage, Record}; +use crate::{mem, RessourceUsage, Record}; pub struct MultiRecordLog { record_log_writer: crate::recordlog::RecordWriter, @@ -355,17 +355,12 @@ impl MultiRecordLog { self.in_mem_queues.last_record(queue) } - /// Returns the quantity of data stored in the in memory queue. - pub fn memory_usage(&self) -> MemoryUsage { - self.in_mem_queues.size() - } - - /// Returns the used disk space. - /// - /// This is typically higher than what [`Self::memory_usage`] reports as records are first - /// marked as truncated, and only get deleted once all other records in the same file are - /// truncated too. - pub fn disk_usage(&self) -> usize { - self.record_log_writer.size() + /// Return the ammount of memory and disk space used by mrecordlog. + pub fn ressource_usage(&self) -> RessourceUsage { + let disk_size = self.record_log_writer.size(); + let (memory_size, memory_capacity) = self.in_mem_queues.size(); + RessourceUsage { + memory_size, memory_capacity, disk_size, + } } } diff --git a/src/tests.rs b/src/tests.rs index 46a3145..6364875 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -323,25 +323,25 @@ fn test_multi_record_size() { let tempdir = tempfile::tempdir().unwrap(); { let mut multi_record_log = MultiRecordLog::open(tempdir.path()).unwrap(); - assert_eq!(multi_record_log.memory_usage().size, 0); - assert_eq!(multi_record_log.memory_usage().capacity, 0); + assert_eq!(multi_record_log.ressource_usage().memory_size, 0); + assert_eq!(multi_record_log.ressource_usage().memory_capacity, 0); multi_record_log.create_queue("queue").unwrap(); - let size_mem_create = multi_record_log.memory_usage(); - assert!(size_mem_create.size > 0); - assert!(size_mem_create.capacity >= size_mem_create.size); + let size_mem_create = multi_record_log.ressource_usage(); + assert!(size_mem_create.memory_size > 0); + assert!(size_mem_create.memory_capacity >= size_mem_create.memory_size); multi_record_log .append_record("queue", None, &b"hello"[..]) .unwrap(); - let size_mem_append = multi_record_log.memory_usage(); - assert!(size_mem_append.size > size_mem_create.size); - assert!(size_mem_append.capacity >= size_mem_append.size); - assert!(size_mem_append.capacity >= size_mem_create.capacity); + let size_mem_append = multi_record_log.ressource_usage(); + assert!(size_mem_append.memory_size > size_mem_create.memory_size); + assert!(size_mem_append.memory_capacity >= size_mem_append.memory_size); + assert!(size_mem_append.memory_capacity >= size_mem_create.memory_capacity); multi_record_log.truncate("queue", 0).unwrap(); - let size_mem_truncate = multi_record_log.memory_usage(); - assert!(size_mem_truncate.size < size_mem_append.size); + let size_mem_truncate = multi_record_log.ressource_usage(); + assert!(size_mem_truncate.memory_size < size_mem_append.memory_size); } } From bd22e93368cff357e2bbbf86fab4cc96c09ed063 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Thu, 7 Mar 2024 20:48:02 +0100 Subject: [PATCH 3/3] rename fields and struct --- src/lib.rs | 10 +++++----- src/mem/queues.rs | 2 +- src/multi_record_log.rs | 16 +++++++++------- src/tests.rs | 22 +++++++++++----------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2dd5132..76ac423 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,15 +28,15 @@ impl<'a> Record<'a> { } } -/// Ressource used by mrecordlog +/// Resources used by mrecordlog #[derive(Clone, Debug, PartialEq, Eq)] -pub struct RessourceUsage { +pub struct ResourceUsage { /// Actual size of the memory used - pub memory_size: usize, + pub memory_used_bytes: usize, /// Capacity allocated, a part of which may be unused right now - pub memory_capacity: usize, + pub memory_allocated_bytes: usize, /// Disk size used - pub disk_size: usize, + pub disk_used_bytes: usize, } #[cfg(test)] diff --git a/src/mem/queues.rs b/src/mem/queues.rs index ebd11c8..3adbbac 100644 --- a/src/mem/queues.rs +++ b/src/mem/queues.rs @@ -6,7 +6,7 @@ use tracing::{info, warn}; use crate::error::{AlreadyExists, AppendError, MissingQueue}; use crate::mem::MemQueue; use crate::rolling::FileNumber; -use crate::{Record}; +use crate::Record; #[derive(Default)] pub(crate) struct MemQueues { diff --git a/src/multi_record_log.rs b/src/multi_record_log.rs index 8ed38d7..44963bc 100644 --- a/src/multi_record_log.rs +++ b/src/multi_record_log.rs @@ -13,7 +13,7 @@ use crate::mem::MemQueue; use crate::record::{MultiPlexedRecord, MultiRecord}; use crate::recordlog::RecordWriter; use crate::rolling::RollingWriter; -use crate::{mem, RessourceUsage, Record}; +use crate::{mem, Record, ResourceUsage}; pub struct MultiRecordLog { record_log_writer: crate::recordlog::RecordWriter, @@ -355,12 +355,14 @@ impl MultiRecordLog { self.in_mem_queues.last_record(queue) } - /// Return the ammount of memory and disk space used by mrecordlog. - pub fn ressource_usage(&self) -> RessourceUsage { - let disk_size = self.record_log_writer.size(); - let (memory_size, memory_capacity) = self.in_mem_queues.size(); - RessourceUsage { - memory_size, memory_capacity, disk_size, + /// Return the amount of memory and disk space used by mrecordlog. + pub fn resource_usage(&self) -> ResourceUsage { + let disk_used_bytes = self.record_log_writer.size(); + let (memory_used_bytes, memory_allocated_bytes) = self.in_mem_queues.size(); + ResourceUsage { + memory_used_bytes, + memory_allocated_bytes, + disk_used_bytes, } } } diff --git a/src/tests.rs b/src/tests.rs index 6364875..7617f45 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -323,25 +323,25 @@ fn test_multi_record_size() { let tempdir = tempfile::tempdir().unwrap(); { let mut multi_record_log = MultiRecordLog::open(tempdir.path()).unwrap(); - assert_eq!(multi_record_log.ressource_usage().memory_size, 0); - assert_eq!(multi_record_log.ressource_usage().memory_capacity, 0); + assert_eq!(multi_record_log.resource_usage().memory_used_bytes, 0); + assert_eq!(multi_record_log.resource_usage().memory_allocated_bytes, 0); multi_record_log.create_queue("queue").unwrap(); - let size_mem_create = multi_record_log.ressource_usage(); - assert!(size_mem_create.memory_size > 0); - assert!(size_mem_create.memory_capacity >= size_mem_create.memory_size); + let size_mem_create = multi_record_log.resource_usage(); + assert!(size_mem_create.memory_used_bytes > 0); + assert!(size_mem_create.memory_allocated_bytes >= size_mem_create.memory_used_bytes); multi_record_log .append_record("queue", None, &b"hello"[..]) .unwrap(); - let size_mem_append = multi_record_log.ressource_usage(); - assert!(size_mem_append.memory_size > size_mem_create.memory_size); - assert!(size_mem_append.memory_capacity >= size_mem_append.memory_size); - assert!(size_mem_append.memory_capacity >= size_mem_create.memory_capacity); + let size_mem_append = multi_record_log.resource_usage(); + assert!(size_mem_append.memory_used_bytes > size_mem_create.memory_used_bytes); + assert!(size_mem_append.memory_allocated_bytes >= size_mem_append.memory_used_bytes); + assert!(size_mem_append.memory_allocated_bytes >= size_mem_create.memory_allocated_bytes); multi_record_log.truncate("queue", 0).unwrap(); - let size_mem_truncate = multi_record_log.ressource_usage(); - assert!(size_mem_truncate.memory_size < size_mem_append.memory_size); + let size_mem_truncate = multi_record_log.resource_usage(); + assert!(size_mem_truncate.memory_used_bytes < size_mem_append.memory_used_bytes); } }