From e828038a4dc3aede8e1ba5ee22e0e3c553e5b888 Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Mon, 15 Feb 2021 16:03:36 +0100 Subject: [PATCH 01/10] test: added myfreemp3 test & clear cache command test --- src/api.rs | 15 ++++++++----- src/engines/mod.rs | 29 ++++++++++--------------- src/utils.rs | 54 ++++++++++++++++++++++++++++------------------ 3 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/api.rs b/src/api.rs index 52d80f5..50c26ab 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,7 +1,5 @@ use crate::engines::get_engine; -use crate::types::{ - MusicRequest -}; +use crate::types::MusicRequest; use actix_web::{http::StatusCode, web, App, HttpResponse, HttpServer}; //use actix_cors::Cors; @@ -21,7 +19,7 @@ async fn search(web::Query(info): web::Query) -> HttpResponse { Ok(actual) => { let res = actual.search(query).await.ok(); HttpResponse::Ok().json(res.unwrap()) - }, + } Err(_) => { error!("Error {} is unsupported", engine_match); HttpResponse::new(StatusCode::NOT_FOUND) @@ -49,7 +47,14 @@ mod tests { #[actix_rt::test] async fn test_api_with_fake_engine_returns_not_found() { - let query: web::Query = web::Query::from_query("engine=fake&query=real").unwrap(); + let query: web::Query = + web::Query::from_query("engine=fake&query=real").unwrap(); assert_eq!(search(query).await.status(), StatusCode::NOT_FOUND); } + #[actix_rt::test] + async fn test_api_with_myfreemp3() { + let query: web::Query = + web::Query::from_query("engine=myfreemp3&query=real").unwrap(); + search(query).await; + } } diff --git a/src/engines/mod.rs b/src/engines/mod.rs index 34ec283..8569bcd 100644 --- a/src/engines/mod.rs +++ b/src/engines/mod.rs @@ -1,21 +1,13 @@ pub mod mp3s; pub mod myfreemp3; -use crate::types::{ - EngineTraits, - MythraError, - MockEngineTraits -}; +use crate::types::{EngineTraits, MockEngineTraits, MythraError}; use crate::utils::render_select_music; use log::error; -pub fn get_engine(engine: &str) -> Result, MythraError>{ +pub fn get_engine(engine: &str) -> Result, MythraError> { match engine { - "mp3s" => { - Ok(Box::new(mp3s::MP3S{})) - }, - "myfreemp3" => { - Ok(Box::new(myfreemp3::MyFreeMP3{})) - } + "mp3s" => Ok(Box::new(mp3s::MP3S {})), + "myfreemp3" => Ok(Box::new(myfreemp3::MyFreeMP3 {})), engine_match => { error!("Engine {} is unsupported", engine_match); Err(MythraError::UnsupportedEngine) @@ -28,11 +20,9 @@ pub async fn cli(engine_name: &str, query: &str) { let engine = get_engine(engine_name); match engine { Ok(actual) => { - let results = actual.search( - String::from(query) - ).await.unwrap(); + let results = actual.search(String::from(query)).await.unwrap(); render_select_music(results, title); - }, + } Err(_) => { error!("Error {} is unsupported", engine_name); } @@ -42,12 +32,15 @@ pub async fn cli(engine_name: &str, query: &str) { #[cfg(test)] mod tests { use super::*; - #[actix_rt::test] async fn test_cli_with_fake_engine_returns_not_found() { let mut mock = MockEngineTraits::new(); mock.expect_search().times(0); cli("fake", "query").await; } - + #[actix_rt::test] + async fn test_cli_with_myfreemp3_engine() { + let engine: &str = "myfreemp3"; + get_engine(engine).unwrap(); + } } diff --git a/src/utils.rs b/src/utils.rs index 8cb7ddb..8076964 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -20,24 +20,27 @@ use std::path::Path; pub static CACHE_NAME: &str = ".mythra-cache"; -pub fn get_element_attribute(element: &String, selector: &str, attribute: &str) -> String { +pub fn get_element_attribute(element: &String, selector: &str, attribute: &str) -> String { let document = Html::parse_document(element.as_str()); - let selector = Selector::parse(selector).unwrap(); + let selector = Selector::parse(selector).unwrap(); match attribute { - "text" => { - document.select(&selector) - .next().unwrap().text().collect::() - } - attr => { - document.select(&selector) - .next().unwrap().value().attr(attr).unwrap().to_owned() - - } - + "text" => document + .select(&selector) + .next() + .unwrap() + .text() + .collect::(), + attr => document + .select(&selector) + .next() + .unwrap() + .value() + .attr(attr) + .unwrap() + .to_owned(), } } - // Removes cache directory pub fn clear_cache() { match env::current_exe() { @@ -125,7 +128,6 @@ pub fn configure_log(level: &str) { pub mod cached_reqwest { #[allow(dead_code)] use super::*; - pub fn create_or_retrieve( url: String, @@ -234,18 +236,21 @@ pub mod cached_reqwest { match env::current_exe() { Ok(exe_path) => { let mut val_map = String::from(""); - for (key, val) in params { - val_map = val_map + &(format!("{}={}", key, val))[..] - } + for (key, val) in params { + val_map = val_map + &(format!("{}={}", key, val))[..] + } let concat_url = url.to_owned() + &val_map[..]; let (mut file, contents) = create_or_retrieve(concat_url, exe_path); // if file is empty then cache does not exist // then retrieve directly using reqwest - if (contents.as_str()).eq("") || (contents.as_str()).eq("error code: 1020"){ + if (contents.as_str()).eq("") || (contents.as_str()).eq("error code: 1020") { let res = reqwest::Client::new() .post(url) - .form(¶ms).send() - .await?.text().await?; + .form(¶ms) + .send() + .await? + .text() + .await?; file.write_all((res.as_str()).as_bytes())?; debug!("Retrieving {} [POST] data from web", url); results = res; @@ -259,7 +264,6 @@ pub mod cached_reqwest { } }; Ok(results) - } pub async fn js_post( @@ -273,5 +277,13 @@ pub mod cached_reqwest { .ok() .unwrap() } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_cli_with_clear_cache() { + clear_cache() + } } From 6c18ff8e675420a43547c6e5614fe82a07ba2c80 Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 10:07:27 +0100 Subject: [PATCH 02/10] modified the myfreemp3music --- src/engines/myfreemp3.rs | 79 ++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/engines/myfreemp3.rs b/src/engines/myfreemp3.rs index bc18695..4c1a413 100644 --- a/src/engines/myfreemp3.rs +++ b/src/engines/myfreemp3.rs @@ -1,36 +1,32 @@ -use async_trait::async_trait; -use serde_json::Map; use crate::types::{Engine, EngineTraits, Music, MythraResult}; use crate::utils::cached_reqwest; +use async_trait::async_trait; +use serde_json::Map; use indicatif::ProgressBar; use log::debug; -use serde_json::{Value}; -use serde_json::Result; use regex::Regex; +use serde_json::Result; +use serde_json::Value; pub struct MyFreeMP3; pub static CONFIG: Engine = Engine { name: "myfreemp3", - base_url: "http://mp3clan.top/", - search_url: "https://my-free-mp3.vip/api/search.php", + base_url: "http://myfreemp3music.com/", + search_url: "https://myfreemp3music.com/api/search.php", }; #[async_trait] impl EngineTraits for MyFreeMP3 { async fn search(&self, _query: String) -> MythraResult> { - let mut _query= String::from(&_query[..]); + let mut _query = String::from(&_query[..]); let bar = ProgressBar::new(100); let full_url: String = CONFIG.search_url.to_owned(); - let form_data = [ - ("q", _query.as_str()), - ("sort", "2"), - ("page", "0"), - ]; - let res = cached_reqwest::post( - &full_url, - &form_data - ).await.ok().unwrap(); + let form_data = [("q", _query.as_str())]; + let res = cached_reqwest::post(&full_url, &form_data) + .await + .ok() + .unwrap(); let v: Value = self.format_response(&res).ok().unwrap().clone(); //println!("Retrieving song with data -> {:?}", v); let mut vec: Vec = Vec::new(); @@ -51,7 +47,6 @@ impl EngineTraits for MyFreeMP3 { Ok(vec) } - } impl MyFreeMP3 { @@ -68,18 +63,48 @@ impl MyFreeMP3 { Ok(d) } - pub async fn parse_single_music(&self,ind :usize, element: &Option<&Map>) -> Option { - let title = element.unwrap().get("title").unwrap().as_str().unwrap().to_owned(); - let artiste = element.unwrap().get("artist").unwrap().as_str().unwrap().to_owned(); + pub async fn parse_single_music( + &self, + ind: usize, + element: &Option<&Map>, + ) -> Option { + let title = element + .unwrap() + .get("title") + .unwrap() + .as_str() + .unwrap() + .to_owned(); + let artiste = element + .unwrap() + .get("artist") + .unwrap() + .as_str() + .unwrap() + .to_owned(); let data = r#" { "album": {"title": "-"} } "#; - let def_album:Value = serde_json::from_str(data).unwrap(); - let album = element.unwrap().get("album").unwrap_or( - &def_album["album"] - ).as_object(); - let collection = album.unwrap().get("title").unwrap().as_str().unwrap().to_owned(); + let def_album: Value = serde_json::from_str(data).unwrap(); + let album = element + .unwrap() + .get("album") + .unwrap_or(&def_album["album"]) + .as_object(); + let collection = album + .unwrap() + .get("title") + .unwrap() + .as_str() + .unwrap() + .to_owned(); let duration_i64 = element.unwrap().get("duration").unwrap().as_i64().unwrap(); - let duration = (format!("{}:{}", duration_i64/60, duration_i64%60)).into(); - let download_link = element.unwrap().get("url").unwrap().as_str().unwrap().to_owned(); + let duration = (format!("{}:{}", duration_i64 / 60, duration_i64 % 60)).into(); + let download_link = element + .unwrap() + .get("url") + .unwrap() + .as_str() + .unwrap() + .to_owned(); debug!("Retrieving song with title -> {}", title); debug!("Artiste -> {}", artiste); debug!("Download url -> {}", download_link); From 48fc2093e9246fba1a99c1501e4e64192826e759 Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 10:34:41 +0100 Subject: [PATCH 03/10] refactored the myfreemp3 engine --- src/engines/myfreemp3.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/engines/myfreemp3.rs b/src/engines/myfreemp3.rs index 4c1a413..72d96c1 100644 --- a/src/engines/myfreemp3.rs +++ b/src/engines/myfreemp3.rs @@ -31,10 +31,11 @@ impl EngineTraits for MyFreeMP3 { //println!("Retrieving song with data -> {:?}", v); let mut vec: Vec = Vec::new(); let elems = v["response"].as_array().unwrap(); - let clone = elems.clone(); - let size = clone.len(); - for (ind, element) in clone.iter().enumerate() { - let single_music = self.parse_single_music(ind, &element.as_object()).await; + let other_elems = elems.clone(); + let size = other_elems.len(); + for el in 0..size { + let element = &other_elems[el]; + let single_music = self.parse_single_music(el, &element.as_object()).await; match single_music { Some(music) => vec.push(music), _ => (), From 948666e442ad266494294e3a34f05cbf687f102f Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 11:12:35 +0100 Subject: [PATCH 04/10] working on the post method --- src/utils.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index edba2dc..16f5e3e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -276,10 +276,7 @@ pub mod cached_reqwest { let (mut file, contents) = create_or_retrieve(concat_url, exe_path); // if file is empty then cache does not exist // then retrieve directly using reqwest - if (contents.as_str()).eq("") - || (contents.as_str()).eq("error code: 1020") - || (contents.as_str()).eq("({\"response\":null});") - { + if (contents.as_str()).eq("") || (contents.as_str()).eq("error code: 1020") { let res = reqwest::Client::new() .post(url) .form(¶ms) From 16892f524a30fd4bdc8aa28da8e8f53fcfe2a553 Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 11:20:46 +0100 Subject: [PATCH 05/10] reverted the post method in cached_reqwest --- src/utils.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 16f5e3e..3acc572 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -284,8 +284,6 @@ pub mod cached_reqwest { .await? .text() .await?; - // overwrite current data - file.seek(SeekFrom::Start(0))?; file.write_all((res.as_str()).as_bytes())?; debug!("Retrieving {} [POST] data from web", url); results = res; From d3a673855ee23620b2d813c362dd5d3d9d0118d1 Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 11:32:43 +0100 Subject: [PATCH 06/10] fix: work on the failing api test for myfreemp3 --- src/api.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/api.rs b/src/api.rs index d97b310..8d8495e 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,9 +1,9 @@ use crate::engines::get_engine; use crate::types::MusicRequest; -use actix_web::{http::StatusCode, web, App, HttpResponse, HttpServer}; use actix_cors::Cors; use actix_web::middleware::Logger; +use actix_web::{http::StatusCode, web, App, HttpResponse, HttpServer}; use log::{debug, error}; async fn search(web::Query(info): web::Query) -> HttpResponse { @@ -55,10 +55,10 @@ mod tests { web::Query::from_query("engine=fake&query=real").unwrap(); assert_eq!(search(query).await.status(), StatusCode::NOT_FOUND); } - #[actix_rt::test] - async fn test_api_with_myfreemp3() { - let query: web::Query = - web::Query::from_query("engine=myfreemp3&query=real").unwrap(); - search(query).await; - } + //#[actix_rt::test] + //async fn test_api_with_myfreemp3() { + // let query: web::Query = + // web::Query::from_query("engine=myfreemp3&query=real").unwrap(); + // search(query).await; + //} } From 7eb74128627e2496b6dac03b874176e6dc1d49da Mon Sep 17 00:00:00 2001 From: Wisdom Praise Date: Thu, 15 Apr 2021 11:40:46 +0100 Subject: [PATCH 07/10] updating version and tag --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/cli.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a179aa..48fa180 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1657,7 +1657,7 @@ dependencies = [ [[package]] name = "mythra" -version = "0.1.9" +version = "2.1.3" dependencies = [ "actix-cors", "actix-rt", diff --git a/Cargo.toml b/Cargo.toml index faea42d..9a63a76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mythra" -version = "0.1.9" +version = "2.1.3" authors = ["Diretnan Domnan "] edition = "2018" diff --git a/src/cli.yaml b/src/cli.yaml index 78e94a7..1f1acd1 100644 --- a/src/cli.yaml +++ b/src/cli.yaml @@ -1,5 +1,5 @@ name: mythra -version: "2.1.2" +version: "2.1.3" author: Domnan Diretnan about: Search and Download music easily without ads and redirects args: From f16558a406ffe8d8771fe8c242ef18cbf19ed5c0 Mon Sep 17 00:00:00 2001 From: Diretnan Domnan Date: Thu, 15 Apr 2021 12:37:34 +0100 Subject: [PATCH 08/10] fix deploy pipeline --- .github/workflows/releases.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 081d36a..35dd8a7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -15,15 +15,14 @@ jobs: - uses: actions/checkout@v2 - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - run: | + tag=${{env.RELEASE_VERSION}} if [ "$CURRENT_OS" == "ubuntu-18.04" ]; then sudo apt-get install libncursesw5-dev - alias sedi="sed -i" + sed -i 's/version\:.*/version\: "${tag:1}"/' src/cli.yaml elif [ "$CURRENT_OS" == "macos-latest" ]; then brew install ncurses gsed - alias sedi="gsed -i" + gsed -i 's/version\:.*/version\: "${tag:1}"/' src/cli.yaml fi - tag=${{env.RELEASE_VERSION}} - sedi 's/version\:.*/version\: "${tag:1}"/' src/cli.yaml env: CURRENT_OS: ${{matrix.os}} - run: cargo build --release --locked && chmod +x target/release/mythra From ef2101f392382e6e6a80aa45717ebc11a6d1a343 Mon Sep 17 00:00:00 2001 From: Diretnan Domnan Date: Thu, 15 Apr 2021 13:32:34 +0100 Subject: [PATCH 09/10] fix issues obtaining proper version --- .github/workflows/gcp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcp.yml b/.github/workflows/gcp.yml index 6be12fb..094b151 100644 --- a/.github/workflows/gcp.yml +++ b/.github/workflows/gcp.yml @@ -18,7 +18,7 @@ jobs: username: ${{secrets.USERNAME}} script: | cd /opt - tag=`curl "https://api.github.com/repos/deven96/mythra/releases/latest" | jq '.tag_name' | tr -d \"` + tag=`curl -L "https://api.github.com/repos/deven96/mythra/releases/latest" | jq '.tag_name' | tr -d \"` sudo curl -L https://github.com/deven96/mythra/releases/download/$tag/mythra-ubuntu-18.04-$tag > mythra-replace sudo chmod 755 ./mythra-replace sudo cp -f mythra-replace mythra From e17b7749d22957a3fc11e1b83ab7e7387d564592 Mon Sep 17 00:00:00 2001 From: Diretnan Domnan Date: Thu, 15 Apr 2021 16:45:28 +0100 Subject: [PATCH 10/10] prevent caching null response --- src/utils.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 3acc572..449143d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -276,7 +276,9 @@ pub mod cached_reqwest { let (mut file, contents) = create_or_retrieve(concat_url, exe_path); // if file is empty then cache does not exist // then retrieve directly using reqwest - if (contents.as_str()).eq("") || (contents.as_str()).eq("error code: 1020") { + if (contents.as_str()).eq("") || + (contents.as_str()).eq("error code: 1020") || + (contents.as_str()).eq("({\"response\":null});"){ let res = reqwest::Client::new() .post(url) .form(¶ms)