0.4.2

sokomishalov · May 27, 2020 · 8c27da1 · 8c27da1
1 parent aefee60
commit 8c27da1
Show file tree

Hide file tree

Showing 8 changed files with 41 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -123,7 +123,7 @@ Maven:
     <dependency>
         <groupId>com.github.sokomishalov.skraper</groupId>
         <artifactId>skrapers</artifactId>
-        <version>0.4.1</version>
+        <version>0.4.2</version>
     </dependency>
 </dependencies>
 ```
@@ -134,7 +134,7 @@ repositories {
     maven { url = uri("http://jitpack.io") }
 }
 dependencies {
-    implementation("com.github.sokomishalov.skraper:skrapers:0.4.1")
+    implementation("com.github.sokomishalov.skraper:skrapers:0.4.2")
 }
 ```
 

diff --git a/cli/pom.xml b/cli/pom.xml
@@ -7,11 +7,11 @@
     <parent>
         <groupId>ru.sokomishalov.skraper</groupId>
         <artifactId>skraper-parent</artifactId>
-        <version>0.4.1</version>
+        <version>0.4.2</version>
     </parent>
 
     <artifactId>cli</artifactId>
-    <version>0.4.1</version>
+    <version>0.4.2</version>
 
     <properties>
         <maven.skip.deploy>true</maven.skip.deploy>
@@ -21,7 +21,7 @@
         <dependency>
             <groupId>ru.sokomishalov.skraper</groupId>
             <artifactId>skrapers</artifactId>
-            <version>0.4.1</version>
+            <version>0.4.2</version>
         </dependency>
         <dependency>
             <groupId>com.xenomachina</groupId>

diff --git a/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt b/cli/src/main/kotlin/ru/sokomishalov/skraper/cli/Main.kt
@@ -42,7 +42,7 @@ import kotlin.text.Charsets.UTF_8
 fun main(args: Array<String>) = mainBody(columns = 100) {
     val parsedArgs = ArgParser(args = args.ifEmpty { arrayOf("--help") }).parseInto(::Args)
 
-    println("${"Skraper".green()} ${"v.0.4.1".magenta()} started")
+    println("${"Skraper".green()} ${"v.0.4.2".magenta()} started")
 
     val posts = runBlocking {
         parsedArgs.skraper.getPosts(

diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle
@@ -37,7 +37,7 @@ repositories {
 }
 
 dependencies {
-    def skraper_version = "0.4.1"
+    def skraper_version = "0.4.2"
 
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation 'androidx.appcompat:appcompat:1.1.0'

diff --git a/pom.xml b/pom.xml
@@ -6,17 +6,17 @@
 
     <groupId>ru.sokomishalov.skraper</groupId>
     <artifactId>skraper-parent</artifactId>
-    <version>0.4.1</version>
+    <version>0.4.2</version>
     <packaging>pom</packaging>
 
     <properties>
         <java.version>1.8</java.version>
         <kotlin.version>1.3.72</kotlin.version>
-        <kotlinx-coroutines.version>1.3.6</kotlinx-coroutines.version>
+        <kotlinx-coroutines.version>1.3.7</kotlinx-coroutines.version>
         <jackson.version>2.11.0</jackson.version>
         <jsoup.version>1.13.1</jsoup.version>
         <reactor-netty.version>0.9.7.RELEASE</reactor-netty.version>
-        <okhttp.version>4.6.0</okhttp.version>
+        <okhttp.version>4.7.2</okhttp.version>
         <spring.version>5.2.6.RELEASE</spring.version>
         <ktor.version>1.3.2</ktor.version>
         <argparser.version>2.0.7</argparser.version>

diff --git a/skrapers/pom.xml b/skrapers/pom.xml
@@ -7,11 +7,11 @@
     <parent>
         <groupId>ru.sokomishalov.skraper</groupId>
         <artifactId>skraper-parent</artifactId>
-        <version>0.4.1</version>
+        <version>0.4.2</version>
     </parent>
 
     <artifactId>skrapers</artifactId>
-    <version>0.4.1</version>
+    <version>0.4.2</version>
 
     <dependencies>
 

diff --git a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/internal/ffmpeg/FfmpegCliRunner.kt
@@ -37,7 +37,7 @@ class FfmpegCliRunner(
 
     private fun checkFfmpegExistence() {
         runBlocking {
-            run(cmd = "-version", timeout = Duration.ofSeconds(1)).let { code ->
+            run(cmd = "-version", timeout = Duration.ofSeconds(2)).let { code ->
                 if (code != 0) System.err.println("`ffmpeg` is not present in OS, some functions may work unreliably")
             }
         }

diff --git a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/facebook/FacebookSkraper.kt
@@ -23,8 +23,10 @@ import ru.sokomishalov.skraper.SkraperClient
 import ru.sokomishalov.skraper.client.jdk.DefaultBlockingSkraperClient
 import ru.sokomishalov.skraper.fetchDocument
 import ru.sokomishalov.skraper.fetchMediaWithOpenGraphMeta
-import ru.sokomishalov.skraper.internal.jsoup.*
-import ru.sokomishalov.skraper.internal.net.queryParams
+import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByAttribute
+import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByAttributeValue
+import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByClass
+import ru.sokomishalov.skraper.internal.jsoup.getFirstElementByTag
 import ru.sokomishalov.skraper.internal.number.div
 import ru.sokomishalov.skraper.internal.serialization.getByPath
 import ru.sokomishalov.skraper.internal.serialization.getInt
@@ -213,36 +215,30 @@ class FacebookSkraper @JvmOverloads constructor(
     }
 
     private fun Element.extractPostMediaItems(): List<Media> {
-        val videoElement = getFirstElementByTag("video")
-
-        return when {
-            videoElement != null -> listOf(Video(
-                    url = getFirstElementByAttributeValueContaining("id", "feed_subtitle")
-                            ?.getFirstElementByTag("a")
-                            ?.attr("href")
-                            ?.let { "${baseUrl}${it}" }
-                            .orEmpty(),
-                    aspectRatio = videoElement
-                            .attr("data-original-aspect-ratio")
-                            ?.toDoubleOrNull()
-            ))
-
-            else -> getFirstElementByClass("uiScaledImageContainer")
-                    ?.getFirstElementByTag("img")
-                    ?.run {
-                        val url = attr("src")?.let {
-                            when {
-                                "safe_image.php" in it -> it.queryParams["url"]
-                                else -> it
-                            }
-                        }.orEmpty()
-
-                        listOf(Image(
-                                url = url,
-                                aspectRatio = attr("width").toDoubleOrNull() / attr("height").toDoubleOrNull()
-                        ))
+        return getElementsByTag("a")
+                .filter { it.hasAttr("ajaxify") && "/stories" !in it.attr("ajaxify") }
+                .mapNotNull { node ->
+                    val videoNode = node.getFirstElementByTag("video")
+                    val imgNode = node.getFirstElementByTag("img")
+
+                    when {
+                        videoNode != null
+                                || "/videos" in node.attr("ajaxify")
+                                || "/watch" in node.attr("ajaxify") -> Video(
+                                url = node
+                                        ?.attr("href")
+                                        ?.let { "${baseUrl}${it}" }
+                                        .orEmpty(),
+                                aspectRatio = videoNode
+                                        ?.attr("data-original-aspect-ratio")
+                                        ?.toDoubleOrNull()
+                        )
+                        imgNode != null -> Image(
+                                url = imgNode.attr("data-src"),
+                                aspectRatio = imgNode.attr("width").toDoubleOrNull() / imgNode.attr("height").toDoubleOrNull()
+                        )
+                        else -> null
                     }
-                    ?: emptyList()
-        }
+                }
     }
 }