From 01b659139fd30d7479ace1d73010e8068056887c Mon Sep 17 00:00:00 2001 From: taevel02 Date: Sun, 11 Feb 2024 20:44:45 +0900 Subject: [PATCH 1/5] fix: handle unexpected panic situations when fetching feeds --- src-tauri/src/producer.rs | 97 ++++++++++++++++++++++++++------------- src-tauri/src/worker.rs | 18 +++++--- 2 files changed, 78 insertions(+), 37 deletions(-) diff --git a/src-tauri/src/producer.rs b/src-tauri/src/producer.rs index 823a6cb..fc73644 100644 --- a/src-tauri/src/producer.rs +++ b/src-tauri/src/producer.rs @@ -13,30 +13,44 @@ use crate::{ syndication::fetch_feed_items, }; -pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Vec { +pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result, String> { let pairs = get_links_to_check(db); let mut inserted = vec![]; - for (feed, link, fetch_old_items) in pairs { - let mut items = fetch_feed_items(&link, proxy).unwrap(); + let mut most_recent_items: Option>> = None; + + for (feed, link, fetch_old_items) in pairs { if !fetch_old_items { - let most_recent_items = get_most_recent_items(db); - if let Some(most_recent) = most_recent_items.get(&feed) { - items.retain(|item| { - item.published_at - .map_or(false, |published_at| published_at > *most_recent) - }); - } else { - items.truncate(1) + if most_recent_items.is_none() { + most_recent_items = match get_most_recent_items(db) { + Ok(items) => Some(items), + Err(err) => return Err(format!("Error fetching most recent items: {}", err)), + }; } } - items.sort_by_key(|x| x.published_at); - inserted.extend(insert_new_items(db, feed, &items)); + match fetch_feed_items(&link, proxy) { + Ok(mut items) => { + if let Some(ref most_recent) = most_recent_items { + if let Some(most_recent) = most_recent.get(&feed) { + items.retain(|item| { + item.published_at + .map_or(false, |published_at| published_at > *most_recent) + }); + } else { + items.truncate(1) + } + } + + items.sort_by_key(|x| x.published_at); + inserted.extend(insert_new_items(db, feed, &items)); + } + Err(err) => return Err(format!("Error fetching feed items: {}", err)), + } } - inserted + Ok(inserted) } fn get_links_to_check(db: &Connection) -> Vec<(i32, String, bool)> { @@ -93,25 +107,46 @@ fn insert_new_items(db: &Connection, feed: i32, items: &[RawItem]) -> Vec HashMap> { - let opt = ItemReadOption { - ids: None, - feed: None, - status: None, - is_saved: None, - order_by: Some(ItemOrder::PublishedDateDesc), - limit: Some(1), - offset: None, - }; +fn get_most_recent_items(db: &Connection) -> Result>, String> { + let mut most_recent_items = HashMap::new(); - let rows = items::read_all(db, &opt).unwrap(); + let feed_ids = match get_all_feed_ids(db) { + Ok(ids) => ids, + Err(err) => return Err(format!("Failed to fetch feed ids: {}", err)), + }; - let mut most_recent_items = HashMap::new(); - for row in rows { - let feed = row.feed.id; - let published_at = row.published_at; - most_recent_items.insert(feed, published_at); + for feed_id in feed_ids { + let opt = ItemReadOption { + ids: None, + feed: Some(feed_id), + status: None, + is_saved: None, + order_by: Some(ItemOrder::PublishedDateDesc), + limit: Some(1), + offset: None, + }; + + match items::read_all(db, &opt) { + Ok(items) => { + if let Some(item) = items.first() { + most_recent_items.insert(item.feed.id, item.published_at); + } + } + Err(err) => { + return Err(format!( + "Failed to fetch items for feed {}: {}", + feed_id, err + )) + } + } } - most_recent_items + Ok(most_recent_items) +} + +fn get_all_feed_ids(db: &Connection) -> Result, String> { + match feeds::read_all(db) { + Ok(feeds) => Ok(feeds.iter().map(|x| x.id).collect()), + Err(err) => Err(err.to_string()), + } } diff --git a/src-tauri/src/worker.rs b/src-tauri/src/worker.rs index 69b895b..3dadcb3 100644 --- a/src-tauri/src/worker.rs +++ b/src-tauri/src/worker.rs @@ -20,13 +20,19 @@ pub fn start(app: &App, app_data_dir: &PathBuf) { let db = open_connection(&app_data_dir).unwrap(); thread::spawn(move || loop { - let inserted = create_new_items(&db, proxy(&db).as_deref()); - if !inserted.is_empty() { - if notification(&db) { - notify(&app_id, &inserted); - } + match create_new_items(&db, proxy(&db).as_deref()) { + Ok(inserted) => { + if !inserted.is_empty() { + if notification(&db) { + notify(&app_id, &inserted); + } - let _ = app_handle.emit_all("feed_updated", ()); + let _ = app_handle.emit_all("feed_updated", ()); + } + } + Err(err) => { + eprintln!("Error fetching new items: {}", err); + } } thread::sleep(time::Duration::from_secs(polling_frequency(&db))); From 07abc4e49a1cd4e7175262998f9c8e9dc09097ec Mon Sep 17 00:00:00 2001 From: taevel02 Date: Sun, 11 Feb 2024 22:21:30 +0900 Subject: [PATCH 2/5] fix: handle errors --- src-tauri/src/error.rs | 6 ++++++ src-tauri/src/producer.rs | 26 ++++++++++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src-tauri/src/error.rs b/src-tauri/src/error.rs index d65c628..cc3c7be 100644 --- a/src-tauri/src/error.rs +++ b/src-tauri/src/error.rs @@ -17,6 +17,12 @@ pub enum Error { #[error("failed to parse syndication feed")] SyndicationParsingFailure, + #[error("failed to fetch feed: {0}")] + FetchFeedFailure(String), + + #[error("failed to fetch feed items: {0}")] + FetchFeedItemsFailure(String), + #[error("empty string")] EmptyString, diff --git a/src-tauri/src/producer.rs b/src-tauri/src/producer.rs index fc73644..bfb6f49 100644 --- a/src-tauri/src/producer.rs +++ b/src-tauri/src/producer.rs @@ -13,7 +13,9 @@ use crate::{ syndication::fetch_feed_items, }; -pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result, String> { +use crate::error::{Error, Result}; + +pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result> { let pairs = get_links_to_check(db); let mut inserted = vec![]; @@ -25,7 +27,7 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result Some(items), - Err(err) => return Err(format!("Error fetching most recent items: {}", err)), + Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), }; } } @@ -46,7 +48,7 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result return Err(format!("Error fetching feed items: {}", err)), + Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), } } @@ -107,13 +109,10 @@ fn insert_new_items(db: &Connection, feed: i32, items: &[RawItem]) -> Vec Result>, String> { +fn get_most_recent_items(db: &Connection) -> Result>> { let mut most_recent_items = HashMap::new(); - let feed_ids = match get_all_feed_ids(db) { - Ok(ids) => ids, - Err(err) => return Err(format!("Failed to fetch feed ids: {}", err)), - }; + let feed_ids = get_all_feed_ids(db)?; for feed_id in feed_ids { let opt = ItemReadOption { @@ -132,21 +131,16 @@ fn get_most_recent_items(db: &Connection) -> Result { - return Err(format!( - "Failed to fetch items for feed {}: {}", - feed_id, err - )) - } + Err(err) => return Err(Error::InvalidValue(err.to_string())), } } Ok(most_recent_items) } -fn get_all_feed_ids(db: &Connection) -> Result, String> { +fn get_all_feed_ids(db: &Connection) -> Result> { match feeds::read_all(db) { Ok(feeds) => Ok(feeds.iter().map(|x| x.id).collect()), - Err(err) => Err(err.to_string()), + Err(err) => Err(Error::FetchFeedFailure(err.to_string())), } } From 30498796ba1b0ba464208294f82549ff5afabf10 Mon Sep 17 00:00:00 2001 From: taevel02 Date: Tue, 13 Feb 2024 17:08:39 +0900 Subject: [PATCH 3/5] fix: reflect reviews --- src-tauri/src/producer.rs | 44 ++++++++++++++------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/src-tauri/src/producer.rs b/src-tauri/src/producer.rs index bfb6f49..f1af067 100644 --- a/src-tauri/src/producer.rs +++ b/src-tauri/src/producer.rs @@ -20,18 +20,18 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result = pairs.iter().map(|(id, _, _)| *id).collect(); + let fetch_old_items_required = pairs.iter().any(|(_, _, fetch_old_items)| !fetch_old_items); + let mut most_recent_items: Option>> = None; + if fetch_old_items_required { + most_recent_items = match get_most_recent_items(db, &feed_ids) { + Ok(items) => Some(items), + Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), + }; + } for (feed, link, fetch_old_items) in pairs { - if !fetch_old_items { - if most_recent_items.is_none() { - most_recent_items = match get_most_recent_items(db) { - Ok(items) => Some(items), - Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), - }; - } - } - match fetch_feed_items(&link, proxy) { Ok(mut items) => { if let Some(ref most_recent) = most_recent_items { @@ -41,7 +41,9 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result *most_recent) }); } else { - items.truncate(1) + if !fetch_old_items { + items.truncate(1) + } } } @@ -109,15 +111,13 @@ fn insert_new_items(db: &Connection, feed: i32, items: &[RawItem]) -> Vec Result>> { +fn get_most_recent_items(db: &Connection, feed_ids: &[i32]) -> Result>> { let mut most_recent_items = HashMap::new(); - let feed_ids = get_all_feed_ids(db)?; - for feed_id in feed_ids { let opt = ItemReadOption { ids: None, - feed: Some(feed_id), + feed: Some(*feed_id), status: None, is_saved: None, order_by: Some(ItemOrder::PublishedDateDesc), @@ -125,22 +125,10 @@ fn get_most_recent_items(db: &Connection) -> Result { - if let Some(item) = items.first() { - most_recent_items.insert(item.feed.id, item.published_at); - } - } - Err(err) => return Err(Error::InvalidValue(err.to_string())), + if let Some(item) = items::read_all(db, &opt)?.first() { + most_recent_items.insert(item.feed.id, item.published_at); } } Ok(most_recent_items) } - -fn get_all_feed_ids(db: &Connection) -> Result> { - match feeds::read_all(db) { - Ok(feeds) => Ok(feeds.iter().map(|x| x.id).collect()), - Err(err) => Err(Error::FetchFeedFailure(err.to_string())), - } -} From df3f58802e567868a28f059fb3ff7584475a5602 Mon Sep 17 00:00:00 2001 From: taevel02 Date: Wed, 14 Feb 2024 19:25:25 +0900 Subject: [PATCH 4/5] refactor: optimize fetching items for feeds --- src-tauri/src/producer.rs | 66 ++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/src-tauri/src/producer.rs b/src-tauri/src/producer.rs index f1af067..83c7804 100644 --- a/src-tauri/src/producer.rs +++ b/src-tauri/src/producer.rs @@ -20,37 +20,44 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result = pairs.iter().map(|(id, _, _)| *id).collect(); - let fetch_old_items_required = pairs.iter().any(|(_, _, fetch_old_items)| !fetch_old_items); + let feed_ids_to_check: Vec = pairs + .iter() + .filter_map(|(id, _, fetch_old_items)| if !fetch_old_items { Some(*id) } else { None }) + .collect(); - let mut most_recent_items: Option>> = None; - if fetch_old_items_required { - most_recent_items = match get_most_recent_items(db, &feed_ids) { - Ok(items) => Some(items), - Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), - }; - } + let most_recent_items = if !feed_ids_to_check.is_empty() { + get_most_recent_items(db, &feed_ids_to_check).unwrap_or_default() + } else { + HashMap::new() + }; for (feed, link, fetch_old_items) in pairs { - match fetch_feed_items(&link, proxy) { - Ok(mut items) => { - if let Some(ref most_recent) = most_recent_items { - if let Some(most_recent) = most_recent.get(&feed) { - items.retain(|item| { - item.published_at - .map_or(false, |published_at| published_at > *most_recent) - }); - } else { - if !fetch_old_items { - items.truncate(1) - } - } - } - - items.sort_by_key(|x| x.published_at); - inserted.extend(insert_new_items(db, feed, &items)); - } + let items = match fetch_feed_items(&link, proxy) { + Ok(items) => items, Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), + }; + + let mut filtered_items = if !fetch_old_items && most_recent_items.get(&feed).is_none() { + items + .into_iter() + .max_by_key(|x| x.published_at) + .into_iter() + .collect() + } else { + items + .into_iter() + .filter(|item| { + most_recent_items.get(&feed).map_or(true, |most_recent| { + item.published_at + .map_or(false, |published_at| published_at > *most_recent) + }) || fetch_old_items + }) + .collect::>() + }; + + if !filtered_items.is_empty() { + filtered_items.sort_by_key(|x| x.published_at); + inserted.extend(insert_new_items(db, feed, &filtered_items)); } } @@ -111,7 +118,10 @@ fn insert_new_items(db: &Connection, feed: i32, items: &[RawItem]) -> Vec Result>> { +fn get_most_recent_items( + db: &Connection, + feed_ids: &[i32], +) -> Result>> { let mut most_recent_items = HashMap::new(); for feed_id in feed_ids { From c2a2461cd67b394c3cc3ec973cfe4576bcc2334a Mon Sep 17 00:00:00 2001 From: taevel02 Date: Sat, 17 Feb 2024 06:28:26 +0900 Subject: [PATCH 5/5] chore: use shortcut for error handling --- src-tauri/src/producer.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src-tauri/src/producer.rs b/src-tauri/src/producer.rs index 83c7804..c5d0f7d 100644 --- a/src-tauri/src/producer.rs +++ b/src-tauri/src/producer.rs @@ -13,7 +13,7 @@ use crate::{ syndication::fetch_feed_items, }; -use crate::error::{Error, Result}; +use crate::error::Result; pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result> { let pairs = get_links_to_check(db); @@ -32,10 +32,7 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result items, - Err(err) => return Err(Error::FetchFeedItemsFailure(err.to_string())), - }; + let items = fetch_feed_items(&link, proxy)?; let mut filtered_items = if !fetch_old_items && most_recent_items.get(&feed).is_none() { items @@ -55,10 +52,8 @@ pub fn create_new_items(db: &Connection, proxy: Option<&str>) -> Result>() }; - if !filtered_items.is_empty() { - filtered_items.sort_by_key(|x| x.published_at); - inserted.extend(insert_new_items(db, feed, &filtered_items)); - } + filtered_items.sort_by_key(|x| x.published_at); + inserted.extend(insert_new_items(db, feed, &filtered_items)); } Ok(inserted)