From 0d37b3f7cdc56941be95d6d975208a3b93a2e0ff Mon Sep 17 00:00:00 2001 From: s1rius Date: Sun, 19 Nov 2023 23:46:08 +0800 Subject: [PATCH] decode string to record array --- Cargo.toml | 23 +- examples/hello_world/Cargo.toml | 14 + .../src/main.rs} | 78 ++-- examples/nest_log/Cargo.toml | 6 +- ezlog-core/Cargo.toml | 3 +- ezlog-core/src/config.rs | 12 + ezlog-core/src/decode.rs | 364 ++++++++++++++---- ezlog-core/src/logger.rs | 16 +- ezlog-core/src/recorder.rs | 20 +- ezlogcli/src/main.rs | 15 +- 10 files changed, 410 insertions(+), 141 deletions(-) create mode 100644 examples/hello_world/Cargo.toml rename examples/{hello_world.rs => hello_world/src/main.rs} (78%) diff --git a/Cargo.toml b/Cargo.toml index 4410cb6..d5c7064 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,7 @@ -[package] -name = "ezlog_space" -version = "0.1.0" -edition = "2021" [workspace] -members = ["ezlogcli", "ezlog-core", "examples/android_preview", "examples/nest_log"] +members = ["ezlogcli", "ezlog-core", "examples/android_preview", "examples/nest_log", "examples/hello_world"] +resolver = "2" # https://github.com/johnthagen/min-sized-rust [profile.release] @@ -12,18 +9,4 @@ opt-level = "z" # Optimize for size. lto = true # Enable Link Time Optimization codegen-units = 1 # Reduce number of codegen units to increase optimizations. # panic = 'abort' # Abort on panic -# strip = true # Strip symbols from binary* - -[dependencies] -ezlog = {path = "ezlog-core", features = ["decode"]} -log = "0.4.17" -time = { version = "0.3", default-features = false, features = ["formatting"] } - -[dev-dependencies] -dirs = "5.0" -rand = "0.8" - -[[example]] -name = "hello_world" -path = "examples/hello_world.rs" -crate-type = ["bin"] \ No newline at end of file +# strip = true # Strip symbols from binary* \ No newline at end of file diff --git a/examples/hello_world/Cargo.toml b/examples/hello_world/Cargo.toml new file mode 100644 index 0000000..220b321 --- /dev/null +++ b/examples/hello_world/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "hello_world" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ezlog = { path = "../../ezlog-core", features = ["log", "decode"]} +log = "0.4" +env_logger = "0" +dirs = "5.0" +time = { version = "0.3", default-features = false, features = ["macros"] } +rand = "0.8" \ No newline at end of file diff --git a/examples/hello_world.rs b/examples/hello_world/src/main.rs similarity index 78% rename from examples/hello_world.rs rename to examples/hello_world/src/main.rs index a168509..2720aa2 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world/src/main.rs @@ -4,10 +4,12 @@ use std::io::{ BufWriter, Cursor, Read, + Write, }; use std::thread; use std::time::Duration; +use ezlog::EZMsg; use ezlog::Level; use ezlog::{ create_log, @@ -19,16 +21,13 @@ use ezlog::{ EventPrinter, Header, }; -use ezlog::{ - EZLogger, - EZMsg, -}; use log::{ debug, error, info, trace, warn, + LevelFilter, }; use rand::Rng; use time::OffsetDateTime; @@ -37,10 +36,10 @@ static EVENT_LISTENER: EventPrinter = EventPrinter; pub fn main() { println!("start"); - ezlog::InitBuilder::new() + let ezlog = ezlog::InitBuilder::new() .with_layer_fn(|msg| { if let EZMsg::Record(recode) = msg { - println!("{}", ezlog::format(&recode)); + println!("{}", ezlog::format(recode)); } }) .with_event_listener(&EVENT_LISTENER) @@ -51,26 +50,45 @@ pub fn main() { create_log(log_config); + log::set_boxed_logger(Box::new(ezlog)) + .map(|()| log::set_max_level(LevelFilter::Trace)) + .unwrap(); + trace!("1. create default log"); debug!("2. debug ez log"); info!("3. now have a log"); warn!("4. test log to file"); error!("5. log complete"); - for i in 0..100 { + for i in 0..10 { trace!("{}{}", i, random_string(300)); } ezlog::flush(ezlog::DEFAULT_LOG_NAME); + + println!("end"); + + thread::sleep(Duration::from_secs(1)); + read_log_file_rewrite(); + + ezlog::set_boxed_callback(Box::new(SimpleCallback)); ezlog::request_log_files_for_date( ezlog::DEFAULT_LOG_NAME, OffsetDateTime::now_utc(), OffsetDateTime::now_utc(), ); - println!("end"); - thread::sleep(Duration::from_secs(1)); - read_log_file_rewrite(); +} + +struct SimpleCallback; + +impl EZLogCallback for SimpleCallback { + fn on_fetch_success(&self, name: &str, date: &str, logs: &[&str]) { + println!("{} {} {}", name, date, logs.join(" ")); + } + fn on_fetch_fail(&self, name: &str, date: &str, err: &str) { + println!("{} {} {}", name, date, err); + } } fn get_config() -> EZLogConfig { @@ -122,30 +140,26 @@ fn read_log_file_rewrite() { let mut writer = BufWriter::new(plaintext_log); - let mut compression = ezlog::create_compress(&log_config); - let mut cryptor = ezlog::create_cryptor(&log_config).unwrap(); + let compression = ezlog::create_compress(&log_config); + let cryptor = ezlog::create_cryptor(&log_config).unwrap(); let header = Header::decode(&mut cursor).unwrap(); - ezlog::decode::decode_body_and_write( - &mut cursor, - &mut writer, - &header.version(), - &mut compression, - &mut cryptor, - &header, - ) - .unwrap(); -} -struct SimpleCallback; + let my_closure = move |data: &Vec, flag: bool| { + writer.write_all(data).unwrap(); + writer.write_all(b"\n").unwrap(); + if flag { + writer.flush().unwrap(); + } + }; -impl EZLogCallback for SimpleCallback { - fn on_fetch_success(&self, name: &str, date: &str, logs: &[&str]) { - println!("fetch success {} {} {}", name, date, logs.join(" ")); - } - - fn on_fetch_fail(&self, name: &str, date: &str, err: &str) { - println!("fetch fail {} {} {}", name, date, err); - } + ezlog::decode::decode_with_fn( + &mut cursor, + header.version(), + &compression, + &cryptor, + &header, + my_closure, + ); } const S: &str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789,.:;!@#$%^&*()_+-"; @@ -158,5 +172,5 @@ fn random_string(length: u32) -> String { let c = chars.nth(index).unwrap(); owned_string.push(c); } - return owned_string; + owned_string } diff --git a/examples/nest_log/Cargo.toml b/examples/nest_log/Cargo.toml index 05e9942..d122d48 100644 --- a/examples/nest_log/Cargo.toml +++ b/examples/nest_log/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "simple_logger" +name = "nest_log" version = "0.1.0" edition = "2021" @@ -7,6 +7,6 @@ edition = "2021" [dependencies] ezlog = { path = "../../ezlog-core", features = ["log"]} -log = "0.4.0" -env_logger = "0.10.0" +log = "0.4" +env_logger = "0" dirs = "5.0" \ No newline at end of file diff --git a/ezlog-core/Cargo.toml b/ezlog-core/Cargo.toml index 6bed7b1..216ec3c 100644 --- a/ezlog-core/Cargo.toml +++ b/ezlog-core/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["lib", "cdylib", "staticlib"] [features] default = [] log = ["dep:log"] -decode = ["dep:aes-gcm"] +decode = ["dep:aes-gcm", "dep:regex", "dep:log"] android_logger = ["log","dep:android_logger"] [dependencies] @@ -35,6 +35,7 @@ thiserror = "1" integer-encoding = "3.0" bitflags = "2.0.1" android_logger = { version = "0.13", optional = true } +regex = { version = "1", optional = true} [target.'cfg(target_os = "android")'.dependencies] jni = "0.21.0" diff --git a/ezlog-core/src/config.rs b/ezlog-core/src/config.rs index f303aed..fd574d7 100644 --- a/ezlog-core/src/config.rs +++ b/ezlog-core/src/config.rs @@ -505,6 +505,18 @@ impl Level { } } +impl std::str::FromStr for Level { + fn from_str(s: &str) -> Result { + let a = LOG_LEVEL_NAMES + .iter() + .position(|&name| name == s) + .and_then(Self::from_usize); + a.ok_or(LogError::Parse("unknown level".to_string())) + } + + type Err = LogError; +} + impl Clone for Level { #[inline] fn clone(&self) -> Level { diff --git a/ezlog-core/src/decode.rs b/ezlog-core/src/decode.rs index 3c45716..69fa7ff 100644 --- a/ezlog-core/src/decode.rs +++ b/ezlog-core/src/decode.rs @@ -1,8 +1,13 @@ -use std::io::{ - self, - BufRead, - Cursor, - Write, +use std::{ + io::{ + self, + BufRead, + Cursor, + Write, + }, + str::FromStr, + sync::mpsc::channel, + time::Duration, }; use byteorder::{ @@ -10,13 +15,21 @@ use byteorder::{ ReadBytesExt, }; use integer_encoding::VarIntReader; +use log::error; +use once_cell::sync::OnceCell; +use regex::Regex; +use time::{ + format_description::well_known::Rfc3339, + OffsetDateTime, +}; use crate::{ errors::LogError, Compress, Cryptor, - EZLogger, + EZRecord, Header, + Level, NonceGenFn, Result, Version, @@ -24,71 +37,70 @@ use crate::{ RECORD_SIGNATURE_START, }; -#[inline] -pub fn decode_logs_count( - logger: &mut EZLogger, - reader: &mut Cursor>, - header: &Header, -) -> Result { - let mut count = 0; - loop { - let position = reader.position(); - match decode_record_from_read( - reader, - &logger.config.version, - &logger.compression, - &logger.cryptor, - header, - position, - ) { - Ok(_) => { - count += 1; - } - Err(e) => match e { - LogError::IoError(err) => { - if err.kind() == io::ErrorKind::UnexpectedEof { - break; - } +pub fn decode_record(vec: &[u8]) -> Result { + static RE: OnceCell> = OnceCell::new(); + let regex = RE.get_or_init(|| Regex::new(r"\[(.*?)\]")); + let record_str = + String::from_utf8(vec.to_vec()).map_err(|e| LogError::Parse(format!("{}", e)))?; + let mut record_builder = EZRecord::builder(); + if let Ok(regex) = regex { + // Search for the first match + if let Some(caps) = regex.captures(&record_str) { + if let Some(matched) = caps.get(1) { + let header = matched.as_str(); + let split: Vec<&str> = header.split_whitespace().collect(); + + let time = split + .first() + .map(|x| { + OffsetDateTime::parse(x, &Rfc3339).unwrap_or(OffsetDateTime::now_utc()) + }) + .unwrap_or(OffsetDateTime::now_utc()); + record_builder.time(time); + + let level = split + .get(1) + .and_then(|x| Level::from_str(x).ok()) + .unwrap_or(Level::Trace); + record_builder.level(level); + + let target = split.get(2).unwrap_or(&"").to_string(); + record_builder.target(target); + + let thread_str = split.get(3).unwrap_or(&""); + let mut thread_info: Vec<&str> = thread_str.split(':').collect(); + if !thread_info.is_empty() { + let thread_id = thread_info.last().unwrap_or(&"").to_string(); + record_builder.thread_id(thread_id.parse::().unwrap_or(0)); + thread_info.pop(); + let thread_name = thread_info.join(":"); + record_builder.thread_name(thread_name); } - LogError::Illegal(_) => break, - _ => continue, - }, - } - } - Ok(count) -} -#[inline] -pub fn decode_body_and_write( - reader: &mut Cursor>, - writer: &mut dyn Write, - version: &Version, - compression: &Option>, - cryptor: &Option>, - header: &Header, -) -> io::Result<()> { - loop { - let position: u64 = reader.position(); - match decode_record_from_read(reader, version, compression, cryptor, header, position) { - Ok(buf) => { - if buf.is_empty() { - break; + #[cfg(feature = "log")] + { + if split.len() == 5 { + let file_info: Vec<&str> = split.last().unwrap_or(&"").split(':').collect(); + if !file_info.is_empty() { + let file_name = file_info.first().unwrap_or(&""); + let line = file_info.get(1).unwrap_or(&""); + + record_builder.file(file_name); + record_builder.line(line.parse::().unwrap_or(0)); + } + }; } - writer.write_all(&buf)?; - writer.write_all(b"\n")?; - } - Err(e) => match e { - LogError::IoError(err) => { - if err.kind() == io::ErrorKind::UnexpectedEof { - break; - } + // header not contains square brackets + // two square brackets and one space + if record_str.len() > header.len() + 3 { + let content = &record_str[header.len() + 3..]; + record_builder.content(content.to_string()); } - LogError::Illegal(_) => break, - _ => continue, - }, + } } } - writer.flush() + + Ok(record_builder.build()) } #[inline] @@ -104,7 +116,7 @@ pub(crate) fn decode_record_from_read( let combine = crate::logger::combine_time_position(header.timestamp.unix_timestamp(), position); let op = Box::new(move |input: &[u8]| crate::logger::xor_slice(input, &combine)); - if header.has_record() && position != header.length() as u64 { + if header.has_record() && !header.is_extra_index(position) { decode_record_content(version, &chunk, compression, cryptor, op) } else { Ok(chunk) @@ -128,7 +140,7 @@ pub(crate) fn decode_record_to_content( reader.read_exact(&mut chunk)?; let end_sign = reader.read_u8()?; if RECORD_SIGNATURE_END != end_sign { - return Err(LogError::Parse("record end sign error".to_string())); + //return Err(LogError::Parse("record end sign error".to_string())); } Ok(chunk) } @@ -184,6 +196,81 @@ pub fn decode_record_content( Ok(buf) } +pub fn decode_with_fn( + reader: &mut Cursor>, + version: &Version, + compression: &Option>, + cryptor: &Option>, + header: &Header, + mut op: F, +) where + F: for<'a> FnMut(&'a Vec, bool), +{ + loop { + let position: u64 = reader.position(); + match decode_record_from_read(reader, version, compression, cryptor, header, position) { + Ok(buf) => { + if buf.is_empty() { + op(&buf, true); + break; + } + op(&buf, false); + } + Err(e) => match e { + LogError::IoError(err) => { + if err.kind() == io::ErrorKind::UnexpectedEof { + op(&vec![], true); + break; + } + } + LogError::Illegal(e) => { + error!(target: "ezlog_decode", "{}", e); + break; + } + _ => { + error!(target: "ezlog_decode", "{}", e); + } + }, + } + } +} + +pub fn decode_with_writer( + cursor: &mut Cursor>, + writer: &mut io::BufWriter, + compression: Option>, + decryptor: Option>, + header: &Header, +) -> Result<()> { + let (tx, rx) = channel(); + let write_closure = move |data: &Vec, flag: bool| { + writer + .write_all(data) + .unwrap_or_else(|e| error!(target: "ezlog_decode", "{}", e)); + writer + .write_all(b"\n") + .unwrap_or_else(|e| error!(target: "ezlog_decode", "{}", e)); + if flag { + writer + .flush() + .unwrap_or_else(|e| error!(target: "ezlog_decode", "{}", e)); + tx.send(()) + .unwrap_or_else(|e| error!(target: "ezlog_decode", "{}", e)) + } + }; + + decode_with_fn( + cursor, + header.version(), + &compression, + &decryptor, + header, + write_closure, + ); + rx.recv_timeout(Duration::from_secs(60 * 5)) + .map_err(|e| LogError::Parse(format!("{}", e))) +} + #[cfg(test)] mod tests { use std::fs; @@ -192,17 +279,23 @@ mod tests { Cursor, Read, }; + use std::sync::mpsc::channel; + use std::time::Duration; - use crate::decode::decode_logs_count; + use time::OffsetDateTime; + + use super::decode_record; + use crate::thread_name; use crate::{ decode, EZLogger, + EZRecord, EZRecordBuilder, Header, }; #[cfg(feature = "decode")] - fn create_all_feature_config() -> crate::EZLogConfig { + fn create_all_feature_config(path: &str) -> crate::EZLogConfig { use crate::CipherKind; use crate::CompressKind; @@ -212,7 +305,7 @@ mod tests { .dir_path( dirs::cache_dir() .unwrap() - .join("ezlog_test") + .join(path) .into_os_string() .into_string() .unwrap(), @@ -250,10 +343,39 @@ mod tests { assert_eq!(vec, decode) } + #[inline] + fn decode_logs_count( + logger: &mut EZLogger, + reader: &mut Cursor>, + header: &Header, + ) -> crate::Result { + let (tx, rx) = channel(); + + let mut count = 0; + let my_closure = |data: &Vec, is_end: bool| { + if data.len() > 0 { + count += 1; + } + if is_end { + tx.send(()).expect("Could not send signal on channel."); + } + }; + crate::decode::decode_with_fn( + reader, + &logger.config.version, + &logger.compression, + &logger.cryptor, + header, + my_closure, + ); + rx.recv().expect("Could not receive from channel."); + Ok(count) + } + #[cfg(feature = "decode")] #[test] fn teset_encode_decode_file() { - let config = create_all_feature_config(); + let config = create_all_feature_config("test_file"); fs::remove_dir_all(&config.dir_path).unwrap_or_default(); let mut logger = EZLogger::new(config.clone()).unwrap(); @@ -288,7 +410,107 @@ mod tests { new_header.recorder_position = Header::length_compat(&config.version) as u32; assert_eq!(header, new_header); let count = decode_logs_count(&mut logger, &mut cursor, &header).unwrap(); + assert_eq!(count, log_count); + drop(logger); + fs::remove_dir_all(&config.dir_path).unwrap_or_default(); + } + + #[inline] + fn decode_array_record( + logger: &mut EZLogger, + reader: &mut Cursor>, + header: &Header, + ) -> crate::Result> { + let mut array: Vec = Vec::new(); + let (tx, rx) = channel(); + + let my_closure = |data: &Vec, is_end: bool| { + if data.len() > 0 { + match decode_record(data) { + Ok(r) => array.push(r), + Err(e) => { + println!("{}", e) + } + } + } + if is_end { + tx.send(()).expect("Could not send signal on channel."); + } + }; + crate::decode::decode_with_fn( + reader, + &logger.config.version, + &logger.compression, + &logger.cryptor, + header, + my_closure, + ); + rx.recv().expect("Could not receive from channel."); + Ok(array) + } + + #[test] + #[cfg(feature = "decode")] + #[cfg(feature = "log")] + pub fn test_decode_to_struct() { + let record = EZRecordBuilder::default() + .time(OffsetDateTime::now_utc()) + .file("demo.rs") + .line(1) + .content("test".to_string()) + .level(crate::Level::Trace) + .target("target".to_string()) + .thread_name(thread_name::get()) + .thread_id(thread_id::get()) + .build(); + + let s = crate::formatter().format(&record).unwrap(); + let record_decode = decode_record(&s).unwrap(); + + assert_eq!(record, record_decode) + } + + #[cfg(feature = "decode")] + #[test] + fn teset_decode_to_array() { + use crate::EZRecord; + + let config = create_all_feature_config("test_array"); + fs::remove_dir_all(&config.dir_path).unwrap_or_default(); + + let mut logger = EZLogger::new(config.clone()).unwrap(); + let log_count = 10; + let mut array: Vec = Vec::new(); + for i in 0..log_count { + let item = EZRecordBuilder::default() + .content(format!("hello world {}", i)) + .time(OffsetDateTime::now_utc() - Duration::from_secs(60 * 60)) + .target("target".to_string()) + .build(); + logger.append(&item.clone()).unwrap(); + array.push(item); + } + logger.flush().unwrap(); + + let (path, _mmap) = &config.create_mmap_file().unwrap(); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) + .unwrap(); + let mut buf = Vec::::new(); + let mut reader = BufReader::new(file); + reader.read_to_end(&mut buf).unwrap(); + assert!(buf.len() > 0); + let mut cursor = Cursor::new(buf); + let header = Header::decode(&mut cursor).unwrap(); + assert!(header.has_record()); + let decode_array = decode_array_record(&mut logger, &mut cursor, &header).unwrap(); + + assert_eq!(array, decode_array); + drop(logger); fs::remove_dir_all(&config.dir_path).unwrap_or_default(); } } diff --git a/ezlog-core/src/logger.rs b/ezlog-core/src/logger.rs index 465f657..78c79d4 100644 --- a/ezlog-core/src/logger.rs +++ b/ezlog-core/src/logger.rs @@ -513,15 +513,23 @@ impl Header { } pub fn has_record_exclude_extra(&self, config: &EZLogConfig) -> bool { - let extra_len = match &config.extra { + // extra write as record + self.recorder_position > (self.length() + self.extra_len(config)) as u32 + } + + #[inline] + fn extra_len(&self, config: &EZLogConfig) -> usize { + match &config.extra { Some(e) => { let record = Vec::from(e.to_owned()); encode_content(record).map(|r| r.len()).unwrap_or(0) } None => 0, - }; - // extra write as record - self.recorder_position > (self.length() + extra_len) as u32 + } + } + + pub fn is_extra_index(&self, position: u64) -> bool { + self.flag.contains(Flags::HAS_EXTRA) && position == self.length() as u64 } pub fn version(&self) -> &Version { diff --git a/ezlog-core/src/recorder.rs b/ezlog-core/src/recorder.rs index 9f6ddd3..da3d8a9 100644 --- a/ezlog-core/src/recorder.rs +++ b/ezlog-core/src/recorder.rs @@ -181,6 +181,20 @@ impl EZRecord { } } +impl PartialEq for EZRecord { + fn eq(&self, other: &Self) -> bool { + self.log_name == other.log_name + && self.level == other.level + && self.target == other.target + && self.time == other.time + && self.thread_id == other.thread_id + && self.thread_name == other.thread_name + && self.content == other.content + && self.file == other.file + && self.line == other.line + } +} + /// [EZRecord]'s builder #[derive(Debug)] pub struct EZRecordBuilder { @@ -235,13 +249,13 @@ impl EZRecordBuilder { } #[cfg(feature = "log")] - fn line(&mut self, line: u32) -> &mut Self { + pub fn line(&mut self, line: u32) -> &mut Self { self.record.line = Some(line); self } #[cfg(feature = "log")] - fn file(&mut self, file: &str) -> &mut Self { + pub fn file(&mut self, file: &str) -> &mut Self { self.record.file = Some(file.to_string()); self } @@ -259,7 +273,7 @@ impl Default for EZRecordBuilder { id: 0, log_name: DEFAULT_LOG_NAME.to_string(), level: Level::Info, - target: "".to_string(), + target: "default".to_string(), time: OffsetDateTime::now_utc(), thread_id: thread_id::get(), thread_name: thread::current().name().unwrap_or("unknown").to_string(), diff --git a/ezlogcli/src/main.rs b/ezlogcli/src/main.rs index 7018944..da5c138 100644 --- a/ezlogcli/src/main.rs +++ b/ezlogcli/src/main.rs @@ -9,7 +9,10 @@ use std::{ path::PathBuf, }; -use anyhow::Context; +use anyhow::{ + anyhow, + Context, +}; use clap::Parser; pub use ezlog::*; use serde::{ @@ -146,16 +149,14 @@ pub fn main() -> anyhow::Result<()> { let mut plain_text_write = BufWriter::new(output_file); - ezlog::decode::decode_body_and_write( + ezlog::decode::decode_with_writer( &mut cursor, &mut plain_text_write, - &config.version, - &compression, - &decryptor, + compression, + decryptor, &header, ) - .unwrap(); - Ok(()) + .map_err(|e| anyhow!(format!("{}", e))) } #[cfg(test)]