From caf1ba68777c874069f9c77919ced13336373c70 Mon Sep 17 00:00:00 2001 From: ilya-bitbucket Date: Fri, 27 Jul 2018 16:23:38 +0300 Subject: [PATCH] Implement homework-3 --- 2396/3/.gitignore | 3 + 2396/3/Gemfile | 10 +++ 2396/3/Gemfile.lock | 83 ++++++++++++++++++++ 2396/3/config.ru | 15 ++++ 2396/3/controllers/application_controller.rb | 11 +++ 2396/3/controllers/posts_controller.rb | 31 ++++++++ 2396/3/helpers/comments_parser.rb | 35 +++++++++ 2396/3/helpers/create_post.rb | 31 ++++++++ 2396/3/helpers/rating_counter.rb | 58 ++++++++++++++ 2396/3/helpers/setting.rb | 21 +++++ 2396/3/lib/workers/post_worker.rb | 13 +++ 2396/3/models/comment.rb | 5 ++ 2396/3/models/post.rb | 7 ++ 2396/3/views/header.erb | 17 ++++ 2396/3/views/layout.erb | 15 ++++ 2396/3/views/navbar.erb | 22 ++++++ 2396/3/views/not_found.erb | 10 +++ 2396/3/views/posts/index.erb | 35 +++++++++ 2396/3/views/posts/new.erb | 11 +++ 2396/3/views/posts/show.erb | 11 +++ 20 files changed, 444 insertions(+) create mode 100644 2396/3/.gitignore create mode 100644 2396/3/Gemfile create mode 100644 2396/3/Gemfile.lock create mode 100644 2396/3/config.ru create mode 100644 2396/3/controllers/application_controller.rb create mode 100644 2396/3/controllers/posts_controller.rb create mode 100644 2396/3/helpers/comments_parser.rb create mode 100644 2396/3/helpers/create_post.rb create mode 100644 2396/3/helpers/rating_counter.rb create mode 100644 2396/3/helpers/setting.rb create mode 100644 2396/3/lib/workers/post_worker.rb create mode 100644 2396/3/models/comment.rb create mode 100644 2396/3/models/post.rb create mode 100644 2396/3/views/header.erb create mode 100644 2396/3/views/layout.erb create mode 100644 2396/3/views/navbar.erb create mode 100644 2396/3/views/not_found.erb create mode 100644 2396/3/views/posts/index.erb create mode 100644 2396/3/views/posts/new.erb create mode 100644 2396/3/views/posts/show.erb diff --git a/2396/3/.gitignore b/2396/3/.gitignore new file mode 100644 index 000000000..95bfbcd4b --- /dev/null +++ b/2396/3/.gitignore @@ -0,0 +1,3 @@ +/config/secrets.yml +config/secrets.yml +app.rb \ No newline at end of file diff --git a/2396/3/Gemfile b/2396/3/Gemfile new file mode 100644 index 000000000..4c53f3f34 --- /dev/null +++ b/2396/3/Gemfile @@ -0,0 +1,10 @@ +source 'https://rubygems.org' + +gem 'mechanize' +gem 'ohm' +gem 'rack' +gem 'redis' +gem 'shotgun' +gem 'sidekiq' +gem 'sinatra' +gem 'thin' diff --git a/2396/3/Gemfile.lock b/2396/3/Gemfile.lock new file mode 100644 index 000000000..e51c19323 --- /dev/null +++ b/2396/3/Gemfile.lock @@ -0,0 +1,83 @@ +GEM + remote: https://rubygems.org/ + specs: + concurrent-ruby (1.0.5) + connection_pool (2.2.2) + daemons (1.2.6) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + eventmachine (1.2.5) + hiredis (0.6.1) + http-cookie (1.0.3) + domain_name (~> 0.5) + mechanize (2.7.6) + domain_name (~> 0.5, >= 0.5.1) + http-cookie (~> 1.0) + mime-types (>= 1.17.2) + net-http-digest_auth (~> 1.1, >= 1.1.1) + net-http-persistent (>= 2.5.2) + nokogiri (~> 1.6) + ntlm-http (~> 0.1, >= 0.1.1) + webrobots (>= 0.0.9, < 0.2) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + mini_portile2 (2.3.0) + mustermann (1.0.2) + nest (3.1.1) + redic + net-http-digest_auth (1.4.1) + net-http-persistent (3.0.0) + connection_pool (~> 2.2) + nokogiri (1.8.4) + mini_portile2 (~> 2.3.0) + ntlm-http (0.1.1) + ohm (3.1.1) + nest (~> 3) + redic (~> 1.5.0) + stal + rack (2.0.5) + rack-protection (2.0.1) + rack + redic (1.5.0) + hiredis + redis (4.0.1) + shotgun (0.9.2) + rack (>= 1.0) + sidekiq (5.1.3) + concurrent-ruby (~> 1.0) + connection_pool (~> 2.2, >= 2.2.0) + rack-protection (>= 1.5.0) + redis (>= 3.3.5, < 5) + sinatra (2.0.1) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.1) + tilt (~> 2.0) + stal (0.3.0) + redic (~> 1.5) + thin (1.7.2) + daemons (~> 1.0, >= 1.0.9) + eventmachine (~> 1.0, >= 1.0.4) + rack (>= 1, < 3) + tilt (2.0.8) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + webrobots (0.1.2) + +PLATFORMS + ruby + +DEPENDENCIES + mechanize + ohm + rack + redis + shotgun + sidekiq + sinatra + thin + +BUNDLED WITH + 1.16.2 diff --git a/2396/3/config.ru b/2396/3/config.ru new file mode 100644 index 000000000..eb707d995 --- /dev/null +++ b/2396/3/config.ru @@ -0,0 +1,15 @@ +require 'sinatra/base' +require 'sidekiq/api' +require 'sidekiq/web' +require 'ohm' + +Dir.glob('./{controllers,helpers,models}/*.rb').sort.each do |file| + require file +end +require './lib/workers/post_worker' +Post.redis = Redic.new('redis://127.0.0.1:6379/0') +Comment.redis = Redic.new('redis://127.0.0.1:6379/1') + +map('/posts') { run PostsController } +map('/sidekiq') { run Sidekiq::Web } +map('/') { run ApplicationController } diff --git a/2396/3/controllers/application_controller.rb b/2396/3/controllers/application_controller.rb new file mode 100644 index 000000000..ee5843613 --- /dev/null +++ b/2396/3/controllers/application_controller.rb @@ -0,0 +1,11 @@ +# This class is the base for my controllers +class ApplicationController < Sinatra::Base + set :views, File.expand_path(File.join(__FILE__, '../../views')) + not_found do + erb :not_found, layout: false + end + + get '/' do + redirect '/posts' + end +end diff --git a/2396/3/controllers/posts_controller.rb b/2396/3/controllers/posts_controller.rb new file mode 100644 index 000000000..48d9de255 --- /dev/null +++ b/2396/3/controllers/posts_controller.rb @@ -0,0 +1,31 @@ +# This is class PostsController +class PostsController < ApplicationController + get '/' do + @posts = Post.all + erb :'posts/index' + end + get '/new' do + erb :'posts/new' + end + + post '/create' do + Post.all.each do |post| + post.delete if post.link == params[:link] + end + PostWorker.perform_async(params[:link]) + sleep 2 + redirect '/' + end + + get '/delete/:id' do + @post = Post.all[params[:id]] + @post.delete + redirect '/' + end + + get '/:id' do + @posts = Post.all + @post = @posts[params[:id]] + erb :'posts/show' + end +end diff --git a/2396/3/helpers/comments_parser.rb b/2396/3/helpers/comments_parser.rb new file mode 100644 index 000000000..99e758561 --- /dev/null +++ b/2396/3/helpers/comments_parser.rb @@ -0,0 +1,35 @@ +require 'json' +require 'mechanize' + +# This is class handling comments for post across API onliner.by +class CommentsParser + LIMIT = 50 + API_PATH = 'https://comments.api.onliner.by/news/tech.post/'.freeze + API_PARAMS = "/comments?limit=#{LIMIT}&_=0.9841189675826583".freeze + attr_reader :agent + + def initialize + @agent = Mechanize.new + end + + def perform(url) + page_post = agent.get(url) + post_id = page_post.search('span.news_view_count').first.values[1] + comment_list = agent.get(API_PATH + post_id + API_PARAMS) + handling_comments(comment_list) + end + + def self.fetch_title(url) + agent = Mechanize.new + agent.get(url).title + end + + private + + def handling_comments(comment_list) + body = comment_list.body + JSON.parse(body)['comments'].each_with_object([]) do |comment, comments| + comments << comment['text'].gsub("\n", '
') + end + end +end diff --git a/2396/3/helpers/create_post.rb b/2396/3/helpers/create_post.rb new file mode 100644 index 000000000..879643202 --- /dev/null +++ b/2396/3/helpers/create_post.rb @@ -0,0 +1,31 @@ +# This is helper for create post from comments +class CreatePost + attr_reader :link + def initialize(link) + @link = link + end + + def perform + create + end + + private + + def create + comments = CommentsParser.new.perform(link) + rating = RatingCounter.new.perform(comments) + post = Post.create title: fetch_title, link: link, + rating: rating.sum / rating.size + create_comments_for_post(post, comments, rating) + end + + def fetch_title + CommentsParser.fetch_title(link) + end + + def create_comments_for_post(post, comments, rating) + comments.zip(rating).each do |obj| + post.comments.add(Comment.create(text: obj.first, rating: obj.last)) + end + end +end diff --git a/2396/3/helpers/rating_counter.rb b/2396/3/helpers/rating_counter.rb new file mode 100644 index 000000000..f6edb5cf0 --- /dev/null +++ b/2396/3/helpers/rating_counter.rb @@ -0,0 +1,58 @@ +require 'net/https' +require 'uri' +require 'json' +require './helpers/setting' +# This is class conects to API AZURE, Analyze sentiment texts and +# fetch score for his +class RatingCounter + KEY_AZURE = Setting.get('key_azure').freeze + URI_BASE = 'https://westcentralus.api.cognitive.microsoft.com/text/analytics/v2.0/sentiment'.freeze + PATH = URI(URI_BASE).freeze + attr_reader :data, :request + + def initialize + @data = { documents: [] } + end + + def perform(comments) + handling_data(comments) + @request = prepare_request + output_data(send_request) + end + + private + + def handling_data(comments) + comments.each_with_index do |comment, index| + @data[:documents] << { 'id' => index.to_s, 'language' => 'ru', + 'text' => comment } + end + end + + def send_request + Net::HTTP.start(PATH.host, PATH.port, + use_ssl: PATH.scheme == 'https') do |http| + http.request(request) + end + end + + def output_data(response) + body = response.body + JSON.parse(body)['documents'].each_with_object([]) do |data, rating| + rating << ((data['score'] * 200).to_i - 100) + end + end + + def headers + heads = {} + heads['Content-Type'] = 'application/json' + heads['Ocp-Apim-Subscription-Key'] = KEY_AZURE + heads + end + + def prepare_request + request = Net::HTTP::Post.new(PATH, headers) + request.body = data.to_json + request + end +end diff --git a/2396/3/helpers/setting.rb b/2396/3/helpers/setting.rb new file mode 100644 index 000000000..4b99a10fe --- /dev/null +++ b/2396/3/helpers/setting.rb @@ -0,0 +1,21 @@ +require 'yaml' + +class Setting + CONFIG_FILE = 'secrets.yml'.freeze + + class << self + def get(key) + load_settings[key.to_s] + end + + def load_settings + YAML.safe_load(File.read(file_path)) + end + + private + + def file_path + "./config/#{CONFIG_FILE}" + end + end +end diff --git a/2396/3/lib/workers/post_worker.rb b/2396/3/lib/workers/post_worker.rb new file mode 100644 index 000000000..875350562 --- /dev/null +++ b/2396/3/lib/workers/post_worker.rb @@ -0,0 +1,13 @@ +# This is class perform jobs in background +class PostWorker + include Sidekiq::Worker + def perform(link) + call(link) + end + + private + + def call(link) + CreatePost.new(link).perform + end +end diff --git a/2396/3/models/comment.rb b/2396/3/models/comment.rb new file mode 100644 index 000000000..8ce36d194 --- /dev/null +++ b/2396/3/models/comment.rb @@ -0,0 +1,5 @@ +# This is model for comments post +class Comment < Ohm::Model + attribute :text + attribute :rating +end diff --git a/2396/3/models/post.rb b/2396/3/models/post.rb new file mode 100644 index 000000000..a84ee6622 --- /dev/null +++ b/2396/3/models/post.rb @@ -0,0 +1,7 @@ +# This is model post +class Post < Ohm::Model + attribute :title + attribute :link + attribute :rating + set :comments, Comment +end diff --git a/2396/3/views/header.erb b/2396/3/views/header.erb new file mode 100644 index 000000000..814fb8cdc --- /dev/null +++ b/2396/3/views/header.erb @@ -0,0 +1,17 @@ + + + + + + + ONLINER + + + + + + + \ No newline at end of file diff --git a/2396/3/views/layout.erb b/2396/3/views/layout.erb new file mode 100644 index 000000000..d28d40812 --- /dev/null +++ b/2396/3/views/layout.erb @@ -0,0 +1,15 @@ +<%= erb :header %> +
+ <%= erb :navbar %> +
+
+
+
<%= yield %>
+
+
+ + + + + + \ No newline at end of file diff --git a/2396/3/views/navbar.erb b/2396/3/views/navbar.erb new file mode 100644 index 000000000..2c6aae364 --- /dev/null +++ b/2396/3/views/navbar.erb @@ -0,0 +1,22 @@ + diff --git a/2396/3/views/not_found.erb b/2396/3/views/not_found.erb new file mode 100644 index 000000000..bb5fe0760 --- /dev/null +++ b/2396/3/views/not_found.erb @@ -0,0 +1,10 @@ + + + Nothing found for 404 + + +
+ +
+ + \ No newline at end of file diff --git a/2396/3/views/posts/index.erb b/2396/3/views/posts/index.erb new file mode 100644 index 000000000..0bb50a330 --- /dev/null +++ b/2396/3/views/posts/index.erb @@ -0,0 +1,35 @@ +<% if @posts.any? %> +

Список статей с рейтингом

+ + + + + + + + <% @posts.each do |post| %> + + + + + + + <% end %> +
Ссылка на статьюРейтинг
+

<%= post.title %>

+ 		+ <%= post.rating %> + + +
+ + +
+ 		+ Удалить +
+<% else %> +
+
+

К сожалению у Вас еще нет статей!

+<% end %> \ No newline at end of file diff --git a/2396/3/views/posts/new.erb b/2396/3/views/posts/new.erb new file mode 100644 index 000000000..0e50a1920 --- /dev/null +++ b/2396/3/views/posts/new.erb @@ -0,0 +1,11 @@ +

Добавьте ссылку на статью для анализа.

+
+
+
+ +
+
+ +
+
+
\ No newline at end of file diff --git a/2396/3/views/posts/show.erb b/2396/3/views/posts/show.erb new file mode 100644 index 000000000..b9d534816 --- /dev/null +++ b/2396/3/views/posts/show.erb @@ -0,0 +1,11 @@ +

<%= @post.title %>

+
+ + + + + + <% @post.comments.each do |comment| %> + + <% end %> +
КомментарийРейтинг

<%= comment.text %>

<%= comment.rating %>
\ No newline at end of file