From ab7088c99c93defacbe73e0a8a6e70ecf76e9e56 Mon Sep 17 00:00:00 2001 From: lienvdsteen Date: Sun, 2 Mar 2014 18:48:02 -0800 Subject: [PATCH 01/11] new_likes and new_visitors --- lib/lonely_coder/profile.rb | 90 ++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 6cc0892..7e6c031 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -1,34 +1,42 @@ # encoding: UTF-8 class OKCupid - + def profile_for(username) Profile.by_username(username, @browser) end - + + def visitors_for(username) + Profile.get_new_visitors(username, @browser) + end + + def likes_for(username) + Profile.get_new_likes(username, @browser) + end + class Profile attr_accessor :username, :match, :friend, :enemy, :location, :age, :sex, :orientation, :single, :small_avatar_url - + # extended profile details - attr_accessor :last_online, :ethnicity, :height, :body_type, :diet, :smokes, - :drinks, :drugs, :religion, :sign, :education, :job, :income, + attr_accessor :last_online, :ethnicity, :height, :body_type, :diet, :smokes, + :drinks, :drugs, :religion, :sign, :education, :job, :income, :offspring, :pets, :speaks, :profile_thumb_urls - - + + # Scraping is never pretty. def self.from_search_result(html) - + username = html.search('span.username').text age, sex, orientation, single = html.search('p.aso').text.split('/') - + percents = html.search('div.percentages') match = percents.search('p.match .percentage').text.to_i friend = percents.search('p.friend .percentage').text.to_i enemy = percents.search('p.enemy .percentage').text.to_i - + location = html.search('p.location').text small_avatar_url = html.search('a.user_image img').attribute('src').value - + OKCupid::Profile.new({ username: username, age: OKCupid.strip(age), @@ -42,24 +50,56 @@ def self.from_search_result(html) small_avatar_url: small_avatar_url }) end - + + def Profile.get_new_likes(username, browser) + html = browser.get("http://www.okcupid.com/who-likes-you") + text = html.search('#whosIntoYouUpgrade .title').text + index = text.index(' people') + likes = text[0, index].to_i + + # todo: get the old likes (do old likes - likes and return new likes) + + return likes + end + + def Profile.get_new_visitors(username, browser) + html = browser.get("http://www.okcupid.com/visitors") + visitors = html.search(".user_list .extra_info .last_visited script") + new_visitors = 0 + previous_timestamp = 1393545600 # todo: get the date when last scraped + + visitors.each { |visitor| + new_visitor = visitor.text + index = new_visitor.index(', ') + date = new_visitor[index + 2, index + 10] + index = date.index(', ') + date = date[0, index].to_i + if (date > previous_timestamp) + new_visitors += 1 + end + } + + # todo: set the date when last scraped + save number of visitors + return new_visitors + end + def Profile.by_username(username, browser) html = browser.get("http://www.okcupid.com/profile/#{username}") - + percents = html.search('#percentages') match = percents.search('span.match').text.to_i friend = percents.search('span.friend').text.to_i enemy = percents.search('span.enemy').text.to_i - + basic = html.search('#aso_loc') age = basic.search('#ajax_age').text sex = basic.search('#ajax_gender').text orientation = basic.search('#ajax_orientation').text single = basic.search('#ajax_status').text location = basic.search('#ajax_location').text - + profile_thumb_urls = html.search('#profile_thumbs img').collect {|img| img.attribute('src').value} - + attributes = { username: username, match: match, @@ -72,34 +112,34 @@ def Profile.by_username(username, browser) single: single, profile_thumb_urls: profile_thumb_urls } - + details_div = html.search('#profile_details dl') - + details_div.each do |node| value = OKCupid.strip(node.search('dd').text) next if value == '—' - + attr_name = node.search('dt').text.downcase.gsub(' ','_') attributes[attr_name] = value end - + self.new(attributes) end - + def initialize(attributes) attributes.each do |attr,val| self.send("#{attr}=", val) end end - + def ==(other) self.username == other.username end - + def eql?(other) self.username == other.username end - + def hash if self.username self.username.hash @@ -108,4 +148,4 @@ def hash end end end -end \ No newline at end of file +end From 784cfda0cd09be8baf743b6811109b8445140918 Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Mon, 3 Mar 2014 17:16:49 -0800 Subject: [PATCH 02/11] merge nico into master --- lib/lonely_coder.rb | 2 ++ lib/lonely_coder/profile.rb | 49 +++++++++++++++++++++++++++++++++++++ lonely_coder.gemspec | 2 ++ 3 files changed, 53 insertions(+) diff --git a/lib/lonely_coder.rb b/lib/lonely_coder.rb index 47e61ab..08695a1 100644 --- a/lib/lonely_coder.rb +++ b/lib/lonely_coder.rb @@ -8,6 +8,8 @@ # `Y' - trek # require 'mechanize' +require 'addressable/uri' +require 'dimensions' class OKCupid BaseUrl = 'http://www.okcupid.com' diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 7e6c031..9bcbcb9 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -1,4 +1,5 @@ # encoding: UTF-8 + class OKCupid def profile_for(username) @@ -13,6 +14,11 @@ def likes_for(username) Profile.get_new_likes(username, @browser) end + + def upload_pic(file, caption) + Profile.upload_picture(file, caption, @browser) + end + class Profile attr_accessor :username, :match, :friend, :enemy, :location, :age, :sex, :orientation, :single, :small_avatar_url @@ -126,6 +132,48 @@ def Profile.by_username(username, browser) self.new(attributes) end + def Profile.upload_picture(file, caption, browser) + + file_dimensions = Dimensions.dimensions(file) + + profile = browser.get('http://www.okcupid.com/profile') + + binding.pry + + authcode = profile.body.match(/authcode['"]?\s*:\s*['"]([\w,;]+?)['"]/)[1] + userid = profile.body.match(/userid['"]?\s*:\s*['"]?(\d+)['"]?/)[1] + + upload_response = browser.post('http://www.okcupid.com/ajaxuploader', { + 'file' => File.new(file) + }) + + picid = upload_response.body.match(/id'\s*:\s*'(\d+)/)[1] + + uri = Addressable::URI.parse('http://www.okcupid.com/photoupload') + uri.query_values = { + :authcode => authcode, + :userid => userid, + :picid => picid, + :width => file_dimensions[0], + :height => file_dimensions[1], + :tn_upper_left_x => 0, + :tn_upper_left_y => 0, + :tn_lower_right_x => file_dimensions[0], + :tn_lower_right_y => file_dimensions[1], + + :caption => caption, + :albumid => 0, + :use_new_upload => 1, + :okc_api => 1, + :'picture.add_ajax' => 1, + } + + uri.to_s + + create_photo = browser.get(uri.to_s) + + end + def initialize(attributes) attributes.each do |attr,val| self.send("#{attr}=", val) @@ -149,3 +197,4 @@ def hash end end end + diff --git a/lonely_coder.gemspec b/lonely_coder.gemspec index 0f51178..b3d73e5 100644 --- a/lonely_coder.gemspec +++ b/lonely_coder.gemspec @@ -10,6 +10,8 @@ Gem::Specification.new do |s| s.description = %q{A gem for interacting with OKCupid as if it had an API.} s.add_dependency 'mechanize', '= 2.0.1' + s.add_dependency 'addressable', '= 2.3.5' + s.add_dependency 'dimensions', '= 1.3.0' s.add_dependency 'activesupport', '>= 3.2.1' s.post_install_message = %q{ From 3f882c643caf899a73c0b31ba470ee843f5a282a Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Mon, 3 Mar 2014 17:41:22 -0800 Subject: [PATCH 03/11] remove pry --- lib/lonely_coder/profile.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 9bcbcb9..ed5c583 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -138,8 +138,6 @@ def Profile.upload_picture(file, caption, browser) profile = browser.get('http://www.okcupid.com/profile') - binding.pry - authcode = profile.body.match(/authcode['"]?\s*:\s*['"]([\w,;]+?)['"]/)[1] userid = profile.body.match(/userid['"]?\s*:\s*['"]?(\d+)['"]?/)[1] From 26b45743642ac9bc3e111dec254df951e8a12134 Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Mon, 3 Mar 2014 17:41:33 -0800 Subject: [PATCH 04/11] add update section --- lib/lonely_coder/profile.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index ed5c583..0c848c3 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -14,6 +14,9 @@ def likes_for(username) Profile.get_new_likes(username, @browser) end + def update_section(section, text) + Profile.update_profile_section(section, text, @browser) + end def upload_pic(file, caption) Profile.upload_picture(file, caption, @browser) @@ -132,6 +135,24 @@ def Profile.by_username(username, browser) self.new(attributes) end + def Profile.update_profile_section(section, text, browser) + section_titles = [ + "My self-summary" + ] + + profile = browser.get('http://www.okcupid.com/profile') + + authcode = profile.body.match(/authcode['"]?\s*:\s*['"]([\w,;]+?)['"]/)[1] + + section_response = browser.post('http://www.okcupid.com/profileedit2', { + :authcode => authcode, + :essay_body => text, + :essay_id => section, + :change_summary => "[title:start]#{section_titles[section]}[title:end][add:start]#{text}[add:end]", + :okc_api => 1 + }) + end + def Profile.upload_picture(file, caption, browser) file_dimensions = Dimensions.dimensions(file) From 51d7d31b1da7fe7adcfa9771dfafab23f0fb3d99 Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Mon, 3 Mar 2014 18:22:22 -0800 Subject: [PATCH 05/11] version bump --- lib/lonely_coder.rb | 2 +- lonely_coder.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lonely_coder.rb b/lib/lonely_coder.rb index 08695a1..d486103 100644 --- a/lib/lonely_coder.rb +++ b/lib/lonely_coder.rb @@ -13,7 +13,7 @@ class OKCupid BaseUrl = 'http://www.okcupid.com' - VERSION = '0.1.5' + VERSION = '0.2.0' def initialize(username=nil, password=nil) @browser = Mechanize.new diff --git a/lonely_coder.gemspec b/lonely_coder.gemspec index b3d73e5..fe4d6ad 100644 --- a/lonely_coder.gemspec +++ b/lonely_coder.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = "lonely_coder" - s.version = '0.1.5.1' + s.version = '0.2.0.0' s.platform = Gem::Platform::RUBY s.authors = ["Trek Glowacki"] s.email = ["trek.glowacki@gmail.com"] From 204e758dd4e75a47cc67da949eac859369f4be59 Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Mon, 3 Mar 2014 22:42:36 -0800 Subject: [PATCH 06/11] added previous timestamp --- lib/lonely_coder/profile.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 0c848c3..53a6602 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -6,8 +6,8 @@ def profile_for(username) Profile.by_username(username, @browser) end - def visitors_for(username) - Profile.get_new_visitors(username, @browser) + def visitors_for(username, previous_timestamp = nil) + Profile.get_new_visitors(username, previous_timestamp, @browser) end def likes_for(username) @@ -71,11 +71,11 @@ def Profile.get_new_likes(username, browser) return likes end - def Profile.get_new_visitors(username, browser) + def Profile.get_new_visitors(username, previous_timestamp = 1393545600, browser) html = browser.get("http://www.okcupid.com/visitors") visitors = html.search(".user_list .extra_info .last_visited script") new_visitors = 0 - previous_timestamp = 1393545600 # todo: get the date when last scraped + # previous_timestamp = 1393545600 # todo: get the date when last scraped visitors.each { |visitor| new_visitor = visitor.text From 39a7eab38bfaca5007b1e65653b55808136c62bf Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Wed, 5 Mar 2014 00:44:54 -0600 Subject: [PATCH 07/11] added essays --- lib/lonely_coder/profile.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 53a6602..e74eb1d 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -29,7 +29,7 @@ class Profile # extended profile details attr_accessor :last_online, :ethnicity, :height, :body_type, :diet, :smokes, :drinks, :drugs, :religion, :sign, :education, :job, :income, - :offspring, :pets, :speaks, :profile_thumb_urls + :offspring, :pets, :speaks, :profile_thumb_urls, :essays # Scraping is never pretty. @@ -109,6 +109,11 @@ def Profile.by_username(username, browser) profile_thumb_urls = html.search('#profile_thumbs img').collect {|img| img.attribute('src').value} + essays = [] + 10.times do |i| + essays[i] = html.search('#essay_text_' + i.to_s).text.strip! + end + attributes = { username: username, match: match, @@ -119,7 +124,8 @@ def Profile.by_username(username, browser) orientation: orientation, location: location, single: single, - profile_thumb_urls: profile_thumb_urls + profile_thumb_urls: profile_thumb_urls, + essays: essays } details_div = html.search('#profile_details dl') From 6826088529337beea0e598057031d5f344880f5a Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Thu, 6 Mar 2014 14:55:20 -0600 Subject: [PATCH 08/11] added further sections and the ability to reference the section by a hash --- lib/lonely_coder/profile.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index e74eb1d..fc234e3 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -143,9 +143,34 @@ def Profile.by_username(username, browser) def Profile.update_profile_section(section, text, browser) section_titles = [ - "My self-summary" + "My self-summary", + "What I’m doing with my life", + "I’m really good at", + "The first things people usually notice about me", + "Favorite books, movies, shows, music, and food", + "The six things I could never do without", + "I spend a lot of time thinking about", + "On a typical Friday night I am", + "The most private thing I’m willing to admit", + "You should message me if" ] + section_titles_hash = { + :self_summary => 0, + :im_doing => 1, + :good_at => 2, + :first_thing => 3, + :favorites => 4, + :six_things => 5, + :think_about => 6, + :private => 7, + :message_me => 8 + } + + if section.class == Symbol + section = section_titles_hash[section] + end + profile = browser.get('http://www.okcupid.com/profile') authcode = profile.body.match(/authcode['"]?\s*:\s*['"]([\w,;]+?)['"]/)[1] From 46837b23878613f364a94df9368e93116e00655a Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Sat, 22 Mar 2014 11:10:48 -0700 Subject: [PATCH 09/11] removing various todos --- lib/lonely_coder/profile.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index fc234e3..1e2bad8 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -66,8 +66,6 @@ def Profile.get_new_likes(username, browser) index = text.index(' people') likes = text[0, index].to_i - # todo: get the old likes (do old likes - likes and return new likes) - return likes end @@ -75,7 +73,7 @@ def Profile.get_new_visitors(username, previous_timestamp = 1393545600, browser) html = browser.get("http://www.okcupid.com/visitors") visitors = html.search(".user_list .extra_info .last_visited script") new_visitors = 0 - # previous_timestamp = 1393545600 # todo: get the date when last scraped + # previous_timestamp = 1393545600 visitors.each { |visitor| new_visitor = visitor.text @@ -88,7 +86,6 @@ def Profile.get_new_visitors(username, previous_timestamp = 1393545600, browser) end } - # todo: set the date when last scraped + save number of visitors return new_visitors end From 43f1d95d424d63716d4a26b601b5521d3f15f77d Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Sat, 22 Mar 2014 11:45:40 -0700 Subject: [PATCH 10/11] corrected visitor fetching --- lib/lonely_coder/profile.rb | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/lonely_coder/profile.rb b/lib/lonely_coder/profile.rb index 1e2bad8..f3990be 100644 --- a/lib/lonely_coder/profile.rb +++ b/lib/lonely_coder/profile.rb @@ -6,7 +6,7 @@ def profile_for(username) Profile.by_username(username, @browser) end - def visitors_for(username, previous_timestamp = nil) + def visitors_for(username, previous_timestamp = 0) Profile.get_new_visitors(username, previous_timestamp, @browser) end @@ -71,17 +71,21 @@ def Profile.get_new_likes(username, browser) def Profile.get_new_visitors(username, previous_timestamp = 1393545600, browser) html = browser.get("http://www.okcupid.com/visitors") - visitors = html.search(".user_list .extra_info .last_visited script") + visitors = html.search(".user_list .user_row_item") + new_visitors = 0 # previous_timestamp = 1393545600 visitors.each { |visitor| - new_visitor = visitor.text - index = new_visitor.index(', ') - date = new_visitor[index + 2, index + 10] - index = date.index(', ') - date = date[0, index].to_i - if (date > previous_timestamp) + begin + timestamp_script = visitor.search(".timestamp script") + timestamp_search = timestamp_script.text.match(/FancyDate\.add\([^,]+?,\s*(\d+)\s*,/) + timestamp = timestamp_search[1] + timestamp = timestamp.to_i + rescue + next + end + if (timestamp > previous_timestamp) new_visitors += 1 end } From 1902c44148776fe0c47f061e821da14666f94894 Mon Sep 17 00:00:00 2001 From: Nick Quinlan Date: Sat, 22 Mar 2014 12:57:05 -0700 Subject: [PATCH 11/11] exposed authentication --- lib/lonely_coder/authentication.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/lonely_coder/authentication.rb b/lib/lonely_coder/authentication.rb index 751125c..ca73ef1 100644 --- a/lib/lonely_coder/authentication.rb +++ b/lib/lonely_coder/authentication.rb @@ -6,6 +6,10 @@ def authenticate(username, password) @authentication = Authentication.new(username, password, @browser) end + def authenticated? + @authentication.success? + end + class Authentication def initialize(username, password, browser) change_to_using_simpler_parser(browser)