Added summarize_existing

Author: bharani91

lib/readability.ex

@@ -109,7 +109,8 @@ defmodule Readability do
 
     html_tree = case url do
       nil -> html_tree
-      _ -> html_tree |> Helper.to_absolute(url)
+      _ -> 
+        html_tree |> Helper.to_absolute(url)
     end
 
     article_tree = html_tree |> ArticleBuilder.build(opts)
@@ -119,6 +120,7 @@ defmodule Readability do
       article_html: readable_html(article_tree),
       article_text: readable_text(article_tree)
     }
+
   end
 
   @doc """

lib/readability/helper.ex

@@ -140,7 +140,7 @@ defmodule Readability.Helper do
       {"img", attrs} ->
         attr_map = Enum.into(attrs, %{})
 
-        src = (Map.get(attr_map, "rel:bf_image_src") || Map.get(attr_map, "data-src") || Map.get(attr_map, "src"))
+        src = (Map.get(attr_map, "rel:bf_image_src") || Map.get(attr_map, "data-original") || Map.get(attr_map, "data-src") || Map.get(attr_map, "src"))
 
         if src do
           src = URI.merge(url, src) |> to_string

mix.exs

@@ -40,10 +40,18 @@ defmodule Readability.Mixfile do
   #
   # Type "mix help deps" for more examples and options
   defp deps do
+<<<<<<< 9cfc6815facb50206a004eae94456543bb37f31a
     [{:floki, "~> 0.13.1"},
      {:httpoison, "~> 0.11.0"},
      {:ex_doc, "~> 0.14", only: :dev},
      {:credo, "~> 0.6.1", only: [:dev, :test]},
+=======
+    [{:floki, "~> 0.11.0"},
+     {:httpoison, "~> 0.11.0"},
+     {:earmark, "~> 0.1", only: :dev},
+     {:ex_doc, "~> 0.11", only: :dev},
+     {:credo, "~> 0.3", only: [:dev, :test]},
+>>>>>>> Added summarize_existing
      {:dialyxir, "~> 0.3", only: [:dev]}
     ]
   end

mix.lock

@@ -1,3 +1,4 @@
+<<<<<<< 9cfc6815facb50206a004eae94456543bb37f31a
 %{"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], []},
   "certifi": {:hex, :certifi, "0.7.0", "861a57f3808f7eb0c2d1802afeaae0fa5de813b0df0979153cbafcd853ababaf", [:rebar3], []},
   "credo": {:hex, :credo, "0.6.1", "a941e2591bd2bd2055dc92b810c174650b40b8290459c89a835af9d59ac4a5f8", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, optional: false]}]},
@@ -7,12 +8,27 @@
   "floki": {:hex, :floki, "0.13.2", "14a4fce7303664b4788e7851a00679c9d8952fa9c8cdd63e11a9e2fa402819b4", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, optional: false]}]},
   "hackney": {:hex, :hackney, "1.6.5", "8c025ee397ac94a184b0743c73b33b96465e85f90a02e210e86df6cbafaa5065", [:rebar3], [{:certifi, "0.7.0", [hex: :certifi, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]},
   "hound": {:hex, :hound, "1.0.3", "bf1859fcb855bf7a3b84c632ba68f04c43bfeb16efebf0080b6c1efb960c30c6", [:mix], [{:hackney, "~> 1.5", [hex: :hackney, optional: false]}, {:poison, ">= 1.4.0", [hex: :poison, optional: false]}]},
+=======
+%{"bunt": {:hex, :bunt, "0.1.6", "5d95a6882f73f3b9969fdfd1953798046664e6f77ec4e486e6fafc7caad97c6f", [:mix], []},
+  "certifi": {:hex, :certifi, "0.7.0", "861a57f3808f7eb0c2d1802afeaae0fa5de813b0df0979153cbafcd853ababaf", [:rebar3], []},
+  "credo": {:hex, :credo, "0.4.5", "5c5daaf50a2a96068c0f21b6fbd382d206702efa8836a946eeab0b8ac25f5f22", [:mix], [{:bunt, "~> 0.1.6", [hex: :bunt, optional: false]}]},
+  "dialyxir": {:hex, :dialyxir, "0.3.5", "eaba092549e044c76f83165978979f60110dc58dd5b92fd952bf2312f64e9b14", [:mix], []},
+  "earmark": {:hex, :earmark, "0.2.1", "ba6d26ceb16106d069b289df66751734802777a3cbb6787026dd800ffeb850f3", [:mix], []},
+  "ex_doc": {:hex, :ex_doc, "0.12.0", "b774aabfede4af31c0301aece12371cbd25995a21bb3d71d66f5c2fe074c603f", [:mix], [{:earmark, "~> 0.2", [hex: :earmark, optional: false]}]},
+  "floki": {:hex, :floki, "0.11.0", "b4532ab64d67225f13f5626e4ba1b8cf3ee9d5bd48017075bb975e1522efd32d", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, optional: false]}]},
+  "hackney": {:hex, :hackney, "1.6.5", "8c025ee397ac94a184b0743c73b33b96465e85f90a02e210e86df6cbafaa5065", [:rebar3], [{:certifi, "0.7.0", [hex: :certifi, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]},
+>>>>>>> Added summarize_existing
   "httpoison": {:hex, :httpoison, "0.11.0", "b9240a9c44fc46fcd8618d17898859ba09a3c1b47210b74316c0ffef10735e76", [:mix], [{:hackney, "~> 1.6.3", [hex: :hackney, optional: false]}]},
   "idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
   "mochiweb": {:hex, :mochiweb, "2.15.0", "e1daac474df07651e5d17cc1e642c4069c7850dc4508d3db7263a0651330aacc", [:rebar3], []},
+<<<<<<< 9cfc6815facb50206a004eae94456543bb37f31a
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], []},
   "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []},
   "wallaby": {:hex, :wallaby, "0.14.0", "476a97a7d592fbd0e3875337c02afeee1ae059e8247791d99b90286ad97917b0", [:mix], [{:httpoison, "~> 0.9", [hex: :httpoison, optional: false]}, {:poison, ">= 1.4.0", [hex: :poison, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}]}}
+=======
+  "mochiweb_html": {:hex, :mochiweb_html, "2.15.0", "d7402e967d7f9f2912f8befa813c37be62d5eeeddbbcb6fe986c44e01460d497", [:rebar3], []},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}}
+>>>>>>> Added summarize_existing

test/readability_test.exs

@@ -3,8 +3,7 @@ defmodule ReadabilityTest do
 
   test "readability for NY Times" do
     html = TestHelper.read_fixture("nytimes.html")
-    opts = [clean_conditionally: false]
-    nytimes = Readability.article(html, opts)
+    nytimes = Readability.article(html)
 
     nytimes_html = Readability.readable_html(nytimes)
     assert nytimes_html =~ ~r/^<div><div class="story-body"><figure id="media-100000004245260" class="media photo lede layout-large-horizontal"><div class="image"><img src="https:\/\/static01.nyt/
@@ -32,8 +31,7 @@ defmodule ReadabilityTest do
 
   test "readability for medium" do
     html = TestHelper.read_fixture("medium.html")
-    opts = [clean_conditionally: false]
-    medium = Readability.article(html, opts)
+    medium = Readability.article(html)
 
     medium_html = Readability.readable_html(medium)
 
@@ -48,8 +46,7 @@ defmodule ReadabilityTest do
 
   test "readability for medium 2" do
     html = TestHelper.read_fixture("medium2.html")
-    opts = [clean_conditionally: false]
-    medium = Readability.article(html, opts)
+    medium = Readability.article(html)
 
     medium_html = Readability.readable_html(medium)
 
@@ -86,6 +83,28 @@ defmodule ReadabilityTest do
     assert pubmed_text =~ ~r/with different mechanisms yielded potent antihypertensive efficacy with safety and decreased plasma BNP levels.$/
   end
 
+  test "summarize_existing for pubmed" do
+    html = TestHelper.read_fixture("pubmed.html")
+    summary = Readability.summarize_existing(html)
+
+    assert summary.article_html =~ ~r/^<div><div class=""><h4>BACKGROUND AND OBJECTIVES: <\/h4><p><abstracttext>Although strict blood pressure/
+    assert summary.article_html =~ ~r/different mechanisms yielded potent antihypertensive efficacy with safety and decreased plasma BNP levels.<\/abstracttext><\/p><\/div><\/div>$/
+
+    assert summary.article_text =~ ~r/^BACKGROUND AND OBJECTIVES: \nAlthough strict blood pressure/
+    assert summary.article_text =~ ~r/with different mechanisms yielded potent antihypertensive efficacy with safety and decreased plasma BNP levels.$/
+  end
+
+  test "summarize_existing with url" do
+    html = TestHelper.read_fixture("pubmed.html")
+    summary = Readability.summarize_existing(html, [url: "http://test.com"])
+
+    assert summary.article_html =~ ~r/^<div><div class=""><h4>BACKGROUND AND OBJECTIVES: <\/h4><p><abstracttext>Although strict blood pressure/
+    assert summary.article_html =~ ~r/different mechanisms yielded potent antihypertensive efficacy with safety and decreased plasma BNP levels.<\/abstracttext><\/p><\/div><\/div>$/
+
+    assert summary.article_text =~ ~r/^BACKGROUND AND OBJECTIVES: \nAlthough strict blood pressure/
+    assert summary.article_text =~ ~r/with different mechanisms yielded potent antihypertensive efficacy with safety and decreased plasma BNP levels.$/
+  end
+
   test "readability for tomaz" do
     html = TestHelper.read_fixture("tomaz.html")
     tomaz = Readability.article(html)
@@ -119,12 +138,12 @@ defmodule ReadabilityTest do
     assert html =~ "So I’m not a big fan out too much automation when it comes to social media, but there are times when it can be useful and save you a lot of time"
   end
 
+
   # test "readability for buzzfeed (url)" do
   #   html = Readability.summarize("https://www.buzzfeed.com/salvadorhernandez/fbi-obtains-passcode-to-iphone-in-new-york-drops-case-agains").article_html
   #   assert html =~ "In New York, as in San Bernardino, an imminent courtroom battle was averted"
   # end
 
-
   # test "readability for newyorker url" do
   #   html = Readability.summarize("http://www.newyorker.com/magazine/2017/01/09/the-vertical-farm").article_html
 
@@ -149,4 +168,5 @@ defmodule ReadabilityTest do
   #   assert html =~ "deportation and murder at the hands of the German Nazis from 1943. In the far south, a terminal,"
   # end
 
+
 end