diff --git a/README.md b/README.md index 11804955..03124211 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Maven: com.github.sokomishalov.skraper skrapers - 0.4.1 + 0.4.2 ``` @@ -134,7 +134,7 @@ repositories { maven { url = uri("http://jitpack.io") } } dependencies { - implementation("com.github.sokomishalov.skraper:skrapers:0.4.1") + implementation("com.github.sokomishalov.skraper:skrapers:0.4.2") } ``` diff --git a/cli/pom.xml b/cli/pom.xml index 196e406f..6b122fe9 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -7,11 +7,11 @@ ru.sokomishalov.skraper skraper-parent - 0.4.1 + 0.4.2 cli - 0.4.1 + 0.4.2 true @@ -21,7 +21,7 @@ ru.sokomishalov.skraper skrapers - 0.4.1 + 0.4.2 com.xenomachina diff --git a/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt b/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt index ff1f53d0..3e9af1a3 100644 --- a/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt +++ b/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt @@ -42,7 +42,7 @@ import kotlin.text.Charsets.UTF_8 fun main(args: Array) = mainBody(columns = 100) { val parsedArgs = ArgParser(args = args.ifEmpty { arrayOf("--help") }).parseInto(::Args) - println("${"Skraper".green()} ${"v.0.4.1".magenta()} started") + println("${"Skraper".green()} ${"v.0.4.2".magenta()} started") val posts = runBlocking { parsedArgs.skraper.getPosts( diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index 4d282289..ff1459ba 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -37,7 +37,7 @@ repositories { } dependencies { - def skraper_version = "0.4.1" + def skraper_version = "0.4.2" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" implementation 'androidx.appcompat:appcompat:1.1.0' diff --git a/pom.xml b/pom.xml index 30246a31..b478d527 100644 --- a/pom.xml +++ b/pom.xml @@ -6,17 +6,17 @@ ru.sokomishalov.skraper skraper-parent - 0.4.1 + 0.4.2 pom 1.8 1.3.72 - 1.3.6 + 1.3.7 2.11.0 1.13.1 0.9.7.RELEASE - 4.6.0 + 4.7.2 5.2.6.RELEASE 1.3.2 2.0.7 diff --git a/skrapers/pom.xml b/skrapers/pom.xml index 30e96b44..d917c7fe 100644 --- a/skrapers/pom.xml +++ b/skrapers/pom.xml @@ -7,11 +7,11 @@ ru.sokomishalov.skraper skraper-parent - 0.4.1 + 0.4.2 skrapers - 0.4.1 + 0.4.2 diff --git a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt index 1913d21c..68fff4ec 100644 --- a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt +++ b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt @@ -37,7 +37,7 @@ class FfmpegCliRunner( private fun checkFfmpegExistence() { runBlocking { - run(cmd = "-version", timeout = Duration.ofSeconds(1)).let { code -> + run(cmd = "-version", timeout = Duration.ofSeconds(2)).let { code -> if (code != 0) System.err.println("`ffmpeg` is not present in OS, some functions may work unreliably") } } diff --git a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt index 86530135..1724aaa7 100644 --- a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt +++ b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt @@ -23,8 +23,10 @@ import ru.sokomishalov.skraper.SkraperClient import ru.sokomishalov.skraper.client.jdk.DefaultBlockingSkraperClient import ru.sokomishalov.skraper.fetchDocument import ru.sokomishalov.skraper.fetchMediaWithOpenGraphMeta -import ru.sokomishalov.skraper.internal.jsoup.* -import ru.sokomishalov.skraper.internal.net.queryParams +import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByAttribute +import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByAttributeValue +import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByClass +import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByTag import ru.sokomishalov.skraper.internal.number.div import ru.sokomishalov.skraper.internal.serialization.getByPath import ru.sokomishalov.skraper.internal.serialization.getInt @@ -213,36 +215,30 @@ class FacebookSkraper @JvmOverloads constructor( } private fun Element.extractPostMediaItems(): List { - val videoElement = getFirstElementByTag("video") - - return when { - videoElement != null -> listOf(Video( - url = getFirstElementByAttributeValueContaining("id", "feed_subtitle") - ?.getFirstElementByTag("a") - ?.attr("href") - ?.let { "${baseUrl}${it}" } - .orEmpty(), - aspectRatio = videoElement - .attr("data-original-aspect-ratio") - ?.toDoubleOrNull() - )) - - else -> getFirstElementByClass("uiScaledImageContainer") - ?.getFirstElementByTag("img") - ?.run { - val url = attr("src")?.let { - when { - "safe_image.php" in it -> it.queryParams["url"] - else -> it - } - }.orEmpty() - - listOf(Image( - url = url, - aspectRatio = attr("width").toDoubleOrNull() / attr("height").toDoubleOrNull() - )) + return getElementsByTag("a") + .filter { it.hasAttr("ajaxify") && "/stories" !in it.attr("ajaxify") } + .mapNotNull { node -> + val videoNode = node.getFirstElementByTag("video") + val imgNode = node.getFirstElementByTag("img") + + when { + videoNode != null + || "/videos" in node.attr("ajaxify") + || "/watch" in node.attr("ajaxify") -> Video( + url = node + ?.attr("href") + ?.let { "${baseUrl}${it}" } + .orEmpty(), + aspectRatio = videoNode + ?.attr("data-original-aspect-ratio") + ?.toDoubleOrNull() + ) + imgNode != null -> Image( + url = imgNode.attr("data-src"), + aspectRatio = imgNode.attr("width").toDoubleOrNull() / imgNode.attr("height").toDoubleOrNull() + ) + else -> null } - ?: emptyList() - } + } } } \ No newline at end of file