Updates

Author: dkhodyriev

.gitignore

@@ -1,4 +1,3 @@
 /target
-/data/.
-data/mydatabase.db-shm
-data/mydatabase.db-wal
+/data
+/.idea

README.md

@@ -30,5 +30,5 @@ What you need to install the software:
 1. **Clone the repository**
 
    ```bash
-   git clone https://github.com/yourusername/rust-web-scraper.git
-   cd rust-web-scraper
+   git clone git@github.com:dhodyrev/async-rust-scraper.git
+   cd async-rust-scraper

data/mydatabase.db

@@ -1,6 +1,7 @@
+use sqlx::SqlitePool;
 use dotenv::dotenv;
 use env_logger::Env;
-use std::env;
+use std::{env, time::Duration};
 use tokio::main;
 
 mod config;
@@ -11,24 +12,39 @@ mod utils;
 use crate::scraper::client::fetch_data;
 use db::operations::insert_data;
 use sqlx::sqlite::SqlitePoolOptions;
+use utils::rate_limiter::rate_limited_operation;
 
-async fn fetch_and_insert_data(pool: &sqlx::SqlitePool, base_url: &str, pages: usize) -> Result<(), Box<dyn std::error::Error>> {
+async fn fetch_and_insert_data(pool: &SqlitePool, base_url: &str, pages: usize) -> Result<(), Box<dyn std::error::Error>> {
     let mut tasks = vec![];
 
     for page in 1..=pages {
         let url = format!("{}/page/{}", base_url, page);
         let pool = pool.clone();
 
         let task = tokio::spawn(async move {
-            match fetch_data(&url).await {
-                Ok(data) => {
-                    if let Err(err) = insert_data(&pool, &data).await {
-                        log::error!("Error inserting data from {}: {}", url, err);
-                    } else {
-                        log::info!("Successfully inserted data from {}", url);
-                    }
+            let result = rate_limited_operation(|| async {
+                match fetch_data(&url).await {
+                    Ok(data) => {
+                        match insert_data(&pool, &data).await {
+                            Ok(_) => {
+                                log::info!("Successfully inserted data from {}", url);
+                                Ok(())
+                            },
+                            Err(err) => {
+                                log::error!("Error inserting data from {}: {}", url, err);
+                                Err(err)
+                            },
+                        }
+                    },
+                    Err(err) => {
+                        log::error!("Error fetching data from {}: {}", url, err);
+                        Err(sqlx::Error::Protocol(err.to_string()))
+                    },
                 }
-                Err(err) => log::error!("Error fetching data from {}: {}", url, err),
+            }, Duration::from_secs(1)).await;
+
+            if let Err(err) = result {
+                log::error!("Error in rate limited operation: {}", err);
             }
         });
 

src/main.rs

@@ -1,12 +1,11 @@
-// utils/rate_limiter.rs
 use tokio::time::{sleep, Duration};
 
-pub async fn rate_limited_operation<F, Fut>(operation: F, delay: Duration)
+pub async fn rate_limited_operation<F, Fut>(operation: F, delay: Duration) -> Result<(), sqlx::Error>
 where
     F: Fn() -> Fut,
-    Fut: std::future::Future<Output = ()>,
+    Fut: std::future::Future<Output = Result<(), sqlx::Error>>,
 {
-    operation().await;
+    let result = operation().await;
     sleep(delay).await;
+    result
 }
-