From 2d202ddbb437f561e8fa6ae904dfccfa262206f1 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 12 Apr 2010 17:58:18 -0500 Subject: [PATCH 01/30] Fixed bug with hotmail skipping 1st contact of each page --- lib/contacts/hotmail.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index bdea819..3b50248 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -79,8 +79,7 @@ def contacts(options = {}) email_match_text_end = Regexp.escape("&") raw_html = resp.body.grep(/(?:e|dn)lk[0-9]+/) - raw_html.delete_at 0 - raw_html.inject do |memo, row| + raw_html.inject(-1) do |memo, row| c_info = row.match(/(e|dn)lk([0-9])+/) # Same contact, or different? @@ -109,6 +108,7 @@ def contacts(options = {}) @contacts << contact end end + return @contacts end end From 8f4fdc6853e14895d5f424882276b9761d6f8ff9 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 12 Apr 2010 18:24:21 -0500 Subject: [PATCH 02/30] Fixed yahoo bug with missing last page and also with reversed first/last names --- lib/contacts/yahoo.rb | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index 97abb0f..71e7fd2 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -62,15 +62,11 @@ def contacts raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - parse data - - parse more_data - if more_data =~ /"TotalABContacts":(\d+)/ total = $1.to_i - ((total / 50)).times do |i| + ((total / 50.0).ceil).times do |i| # now proceed with the new ".crumb" parameter to get the csv data - url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i+2}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) + url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) http = open_http(url) resp, more_data = http.get("#{url.path}?#{url.query}", "Cookie" => @cookies, @@ -81,7 +77,7 @@ def contacts if resp.code_type != Net::HTTPOK raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - + parse more_data end end @@ -94,13 +90,11 @@ def contacts def parse(data, options={}) @contacts ||= [] - if data =~ /var InitialContacts = (\[.*?\]);/ - @contacts += Contacts.parse_json($1).select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} - elsif data =~ /^\{"response":/ - @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} - else - @contacts - end + @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map do |contact| + name = contact["contactName"].split(",") + [[name.pop, name.join(",")].join(" "), contact["email"]] + end if data =~ /^\{"response":/ + @contacts end end From 39bf8127a2d7fe7a2dc8579e97323d294fcbc784 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 12 Apr 2010 18:24:52 -0500 Subject: [PATCH 03/30] Added new gemspec --- glennfu-contacts.gemspec | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 glennfu-contacts.gemspec diff --git a/glennfu-contacts.gemspec b/glennfu-contacts.gemspec new file mode 100644 index 0000000..e661ad1 --- /dev/null +++ b/glennfu-contacts.gemspec @@ -0,0 +1,14 @@ +Gem::Specification.new do |s| + s.name = "glennfu-contacts" + s.version = "1.2.3.1" + s.date = "2010-04-12" + s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.email = "glenn@glennfu.com" + s.homepage = "http://github.com/glennfu/contacts" + s.description = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.has_rdoc = false + s.authors = ["Glenn Sidney", "Lucas Carlson"] + s.files = ["LICENSE", "Rakefile", "README", "examples/grab_contacts.rb", "lib/contacts.rb", "lib/contacts/base.rb", "lib/contacts/json_picker.rb", "lib/contacts/gmail.rb", "lib/contacts/aol.rb", "lib/contacts/hotmail.rb", "lib/contacts/plaxo.rb", "lib/contacts/yahoo.rb"] + s.add_dependency("json", ">= 1.1.1") + s.add_dependency('gdata', '>= 1.1.1') +end \ No newline at end of file From 8ef50282e480617254c45ad00b43a19107b78991 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Fri, 13 Aug 2010 12:14:25 -0500 Subject: [PATCH 04/30] Fixes Yahoo returning nil instead of [] when finding 0 contacts --- lib/contacts/yahoo.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index 558fc8e..d3f09e5 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -62,6 +62,8 @@ def contacts raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end + @contacts ||= [] + if more_data =~ /"TotalABContacts":(\d+)/ total = $1.to_i ((total / 50.0).ceil).times do |i| From 25794ec0d7a764c28c642a4c4fcac8536221a8e5 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 27 Sep 2010 15:14:09 -0500 Subject: [PATCH 05/30] Fixes broken hotmail imports --- lib/contacts/hotmail.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 827c570..1f7666f 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -45,6 +45,7 @@ def real_connect data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] end + @domain = URI.parse(old_url).host @cookies = cookies rescue AuthenticationError => m @@ -76,7 +77,7 @@ def contacts(options = {}) resp, data = http.get(get_contact_list_url(index), "Cookie" => @cookies) email_match_text_beginning = Regexp.escape("http://m.mail.live.com/?rru=compose&to=") - email_match_text_end = Regexp.escape("&") + email_match_text_end = Regexp.escape("&ru=") raw_html = resp.body.split(" ").grep(/(?:e|dn)lk[0-9]+/) @@ -105,7 +106,7 @@ def contacts(options = {}) build_contacts.each do |contact| unless contact[1].nil? # Only return contacts with email addresses - contact[1] = CGI::unescape(contact[1]) + contact[1] = CGI::unescape CGI::unescape(contact[1]) @contacts << contact end end From a7b6b2b2894387346bf1e8d070a04e980f7de575 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Tue, 30 Nov 2010 12:01:16 -0600 Subject: [PATCH 06/30] Thanks to mrich54907 - fixed hotmail import bug --- lib/contacts/hotmail.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 1f7666f..b3c41b4 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -90,7 +90,7 @@ def contacts(options = {}) # Grab info case c_info[1] when "e" # Email - build_contacts.last[1] = row.match(/#{email_match_text_beginning}(.*)#{email_match_text_end}/)[1] + build_contacts.last[1] = row.match(/#{email_match_text_beginning}(.*?)#{email_match_text_end}/)[1] when "dn" # Name build_contacts.last[0] = row.match(/]*>(.+)<\/a>/)[1] end From 364d76cf938205f64b63554d9eef04f9fb363426 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Tue, 30 Nov 2010 16:11:00 -0600 Subject: [PATCH 07/30] Attempts to fix some Yahoo issues... still failing incorrectly with wrong password error sometimes --- lib/contacts/yahoo.rb | 63 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index 5b27d36..7f90f0e 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -5,17 +5,24 @@ class Yahoo < Base ADDRESS_BOOK_URL = "http://address.mail.yahoo.com/?.rand=430244936" CONTACT_LIST_URL = "http://address.mail.yahoo.com/?_src=&_crumb=crumb&sortfield=3&bucket=1&scroll=1&VPC=social_list&.r=time" PROTOCOL_ERROR = "Yahoo has changed its protocols, please upgrade this library first. If that does not work, dive into the code and submit a patch at http://github.com/cardmagic/contacts" - - def real_connect + INVALID_PASS = '
Invalid ID or password.
Please try again using your full Yahoo! ID.
' + NOT_YET_TAKEN = '
This ID is not yet taken.
Are you trying to ' + + def real_connect(attempt_count=0) postdata = ".tries=2&.src=ym&.md5=&.hash=&.js=&.last=&promo=&.intl=us&.bypass=" postdata += "&.partner=&.u=4eo6isd23l8r3&.v=0&.challenge=gsMsEcoZP7km3N3NeI4mX" postdata += "kGB7zMV&.yplus=&.emailCode=&pkg=&stepid=&.ev=&hasMsgr=1&.chkP=Y&." postdata += "done=#{CGI.escape(URL)}&login=#{CGI.escape(login)}&passwd=#{CGI.escape(password)}" - + data, resp, cookies, forward = post(LOGIN_URL, postdata) - - if data.index("Invalid ID or password") || data.index("This ID is not yet taken") - raise AuthenticationError, "Username and password do not match" + + if data.index(INVALID_PASS) || data.index(NOT_YET_TAKEN) + if attempt_count < 1 + sleep(5) + return real_connect(attempt_count + 1) + else + raise AuthenticationError, "Username and password do not match" + end elsif data.index("Sign in") && data.index("to Yahoo!") raise AuthenticationError, "Required field must not be blank" elsif !data.match(/uncompressed\/chunked/) @@ -23,20 +30,18 @@ def real_connect elsif cookies == "" raise ConnectionError, PROTOCOL_ERROR end - + data, resp, cookies, forward = get(forward, cookies, LOGIN_URL) - + if resp.code_type != Net::HTTPOK raise ConnectionError, PROTOCOL_ERROR end - + @cookies = cookies end - - def contacts + + def contacts return @contacts if @contacts - @contacts = [] - if connected? # first, get the addressbook site with the new crumb parameter url = URI.parse(address_book_url) @@ -63,12 +68,16 @@ def contacts if resp.code_type != Net::HTTPOK raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - + + parse data + + parse more_data + if more_data =~ /"TotalABContacts":(\d+)/ total = $1.to_i - ((total / 50.0).ceil).times do |i| + ((total / 50)).times do |i| # now proceed with the new ".crumb" parameter to get the csv data - url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) + url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i+1}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) http = open_http(url) resp, more_data = http.get("#{url.path}?#{url.query}", "Cookie" => @cookies, @@ -79,27 +88,29 @@ def contacts if resp.code_type != Net::HTTPOK raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - + parse more_data end end - + @contacts end end private - + def parse(data, options={}) @contacts ||= [] - @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map do |contact| - name = contact["contactName"].split(",") - [[name.pop, name.join(",")].join(" ").strip, contact["email"]] - end if data =~ /^\{"response":/ - @contacts + if data =~ /var InitialContacts = (\[.*?\]);/ + @contacts += Contacts.parse_json($1).select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} + elsif data =~ /^\{"response":/ + @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} + else + @contacts + end end - + end TYPES[:yahoo] = Yahoo -end +end \ No newline at end of file From f0c8e6894ade872b4cf515c57285bcfdacd1b2e1 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Thu, 12 May 2011 16:21:01 -0500 Subject: [PATCH 08/30] Incorporated AOL fix from cbaker --- lib/contacts/aol.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 5336ae6..a36d895 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -6,10 +6,10 @@ class Aol < Base LOGIN_URL = "https://my.screenname.aol.com/_cqr/login/login.psp" LOGIN_REFERER_URL = "http://webmail.aol.com/" LOGIN_REFERER_PATH = "sitedomain=sns.webmail.aol.com&lang=en&locale=us&authLev=0&uitype=mini&loginId=&redirType=js&xchk=false" - AOL_NUM = "29970-343" # this seems to change each time they change the protocol + AOL_NUM = "32992-111" # this seems to change each time they change the protocol - CONTACT_LIST_URL = "http://webmail.aol.com/#{AOL_NUM}/aim-2/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" - CONTACT_LIST_CSV_URL = "http://webmail.aol.com/#{AOL_NUM}/aim-2/en-us/Lite/ABExport.aspx?command=all" + CONTACT_LIST_URL = "http://mail.aol.com/#{AOL_NUM}/aol-6/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" + CONTACT_LIST_CSV_URL = "http://mail.aol.com/#{AOL_NUM}/aol-6/en-us/Lite/ABExport.aspx?command=all" PROTOCOL_ERROR = "AOL has changed its protocols, please upgrade this library first. If that does not work, dive into the code and submit a patch at http://github.com/cardmagic/contacts" def real_connect From ddce1902365751958059121cca6faeefef667d8d Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Thu, 12 May 2011 16:22:25 -0500 Subject: [PATCH 09/30] Fixed aol not recognizing bad password error --- lib/contacts/aol.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index a36d895..0450b16 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -80,7 +80,7 @@ def real_connect data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] end - if data.index("Invalid Username or Password. Please try again.") + if data.index("Incorrect Username or Password.") raise AuthenticationError, "Username and password do not match" elsif data.index("Required field must not be blank") raise AuthenticationError, "Login and password must not be blank" From 5193d46375cb019011801b162a1d806e2265ae76 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 23 May 2011 12:23:44 -0400 Subject: [PATCH 10/30] No more hard-coding the AOL_NUM as it's changing too frequently, scrape it instead --- lib/contacts/aol.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 0450b16..2bca129 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -6,10 +6,6 @@ class Aol < Base LOGIN_URL = "https://my.screenname.aol.com/_cqr/login/login.psp" LOGIN_REFERER_URL = "http://webmail.aol.com/" LOGIN_REFERER_PATH = "sitedomain=sns.webmail.aol.com&lang=en&locale=us&authLev=0&uitype=mini&loginId=&redirType=js&xchk=false" - AOL_NUM = "32992-111" # this seems to change each time they change the protocol - - CONTACT_LIST_URL = "http://mail.aol.com/#{AOL_NUM}/aol-6/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" - CONTACT_LIST_CSV_URL = "http://mail.aol.com/#{AOL_NUM}/aol-6/en-us/Lite/ABExport.aspx?command=all" PROTOCOL_ERROR = "AOL has changed its protocols, please upgrade this library first. If that does not work, dive into the code and submit a patch at http://github.com/cardmagic/contacts" def real_connect @@ -91,7 +87,11 @@ def real_connect elsif cookies == "" raise ConnectionError, PROTOCOL_ERROR end - + + @aol_num = data.match(/URL=\"http:\/\/mail.aol.com\/(.*)\/aol-6\/en-us\/common\/error.aspx\?/)[1] + @contact_list_url = "http://mail.aol.com/#{@aol_num}/aol-6/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" + @contact_list_csv_url = "http://mail.aol.com/#{@aol_num}/aol-6/en-us/Lite/ABExport.aspx?command=all" + @cookies = cookies end @@ -103,7 +103,7 @@ def contacts return @contacts if @contacts if connected? - data, resp, cookies, forward, old_url = get(CONTACT_LIST_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] @@ -119,7 +119,7 @@ def contacts postdata["user"] = input.attributes["value"] if input.attributes["name"] == "user" end - data, resp, cookies, forward, old_url = get(CONTACT_LIST_CSV_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_csv_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] From 0806eb94d292e2947ed9b8ec1f072a920d0bdf7c Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 23 May 2011 12:25:56 -0400 Subject: [PATCH 11/30] Whoops forgot to step the gem version --- contacts.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contacts.gemspec b/contacts.gemspec index 13ef4e7..8d365df 100644 --- a/contacts.gemspec +++ b/contacts.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "contacts" - s.version = "1.2.4" + s.version = "1.2.5" s.date = "2010-07-06" s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." s.email = "lucas@rufy.com" From c8bf7e58649017218737962ab5c4636585c60b19 Mon Sep 17 00:00:00 2001 From: Glenn Sidney Date: Mon, 13 Jun 2011 13:25:21 -0500 Subject: [PATCH 12/30] Fixes AOL import bug when a contact's name as a double-quote in it --- contacts.gemspec | 2 +- geminstaller.yml | 2 ++ glennfu-contacts.gemspec | 15 +++++++++++++++ lib/contacts/aol.rb | 18 +++++++++++++----- 4 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 glennfu-contacts.gemspec diff --git a/contacts.gemspec b/contacts.gemspec index 8d365df..288869b 100644 --- a/contacts.gemspec +++ b/contacts.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "contacts" - s.version = "1.2.5" + s.version = "1.2.6" s.date = "2010-07-06" s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." s.email = "lucas@rufy.com" diff --git a/geminstaller.yml b/geminstaller.yml index 26448aa..9f3b0e4 100644 --- a/geminstaller.yml +++ b/geminstaller.yml @@ -6,3 +6,5 @@ gems: version: >= 1.1.1 - name: gdata version: >= 1.1.1 + - name: fastercsv + version: >= 1.5.4 diff --git a/glennfu-contacts.gemspec b/glennfu-contacts.gemspec new file mode 100644 index 0000000..caa8b03 --- /dev/null +++ b/glennfu-contacts.gemspec @@ -0,0 +1,15 @@ +Gem::Specification.new do |s| + s.name = "glennfu-contacts" + s.version = "1.2.6" + s.date = "2011-06-13" + s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.email = "glenn@glennfu.com" + s.homepage = "http://github.com/glennfu/contacts" + s.description = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.has_rdoc = false + s.authors = ["Glenn Sidney", "Lucas Carlson"] + s.files = ["LICENSE", "Rakefile", "README", "examples/grab_contacts.rb", "lib/contacts.rb", "lib/contacts/base.rb", "lib/contacts/json_picker.rb", "lib/contacts/gmail.rb", "lib/contacts/aol.rb", "lib/contacts/hotmail.rb", "lib/contacts/plaxo.rb", "lib/contacts/yahoo.rb"] + s.add_dependency("json", ">= 1.1.1") + s.add_dependency('gdata', '>= 1.1.1') + s.add_dependency('fastercsv', '>= 1.5.4') +end \ No newline at end of file diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 2bca129..c64280b 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -135,11 +135,19 @@ def contacts private def parse(data, options={}) - data = CSV::Reader.parse(data) - col_names = data.shift - @contacts = data.map do |person| - ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? - end.compact + begin + @contacts = [] + FasterCSV.parse(data) do |person| + @contacts << ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? + end + @contacts + rescue + parsed_data = CSV::Reader.parse(data) + col_names = parsed_data.shift + @contacts = parsed_data.map do |person| + ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? + end.compact + end end def h_to_query_string(hash) From da176e0aa719b2def5201307643f168b0abee88f Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 16:58:39 -0700 Subject: [PATCH 13/30] removing a ruby 1.9 incompatibility due to CSV::Reader deprecation --- lib/contacts/aol.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 5336ae6..b108dbd 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -135,7 +135,11 @@ def contacts private def parse(data, options={}) - data = CSV::Reader.parse(data) + if CSV.const_defined? :Reader + data = CSV::Reader.parse(data) + else + data = CSV.parse(data) + end col_names = data.shift @contacts = data.map do |person| ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? @@ -151,4 +155,4 @@ def h_to_query_string(hash) end TYPES[:aol] = Aol -end \ No newline at end of file +end From 3026fcaa0bd575f69265a08a2ab30f6bc172594e Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 17:18:31 -0700 Subject: [PATCH 14/30] new CSV.parse is a subset of former CSV::Reader.parse. doing the dispatch here. --- lib/contacts/aol.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index b108dbd..804f259 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -138,7 +138,11 @@ def parse(data, options={}) if CSV.const_defined? :Reader data = CSV::Reader.parse(data) else - data = CSV.parse(data) + if data.is_a?(String) + data = CSV.parse(data) + else + data = CSV.read(data) + end end col_names = data.shift @contacts = data.map do |person| From 7cdb21d901bd1be3754596f20e63b2b3058e6f4b Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 17:35:55 -0700 Subject: [PATCH 15/30] adding contacts_data to make debugging the malformed CSV easier --- lib/contacts/aol.rb | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 804f259..e1aa82a 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -132,6 +132,45 @@ def contacts parse data end end + + def contacts_data + postdata = { + "file" => 'contacts', + "fileType" => 'csv' + } + + return @contacts if @contacts + if connected? + data, resp, cookies, forward, old_url = get(CONTACT_LIST_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + # parse data and grab + doc = Hpricot(data) + (doc/:input).each do |input| + postdata["user"] = input.attributes["value"] if input.attributes["name"] == "user" + end + + data, resp, cookies, forward, old_url = get(CONTACT_LIST_CSV_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + if data.include?("error.gif") + raise AuthenticationError, "Account invalid" + end + + data + end + end + private def parse(data, options={}) From edc4d057410d9d0eed33530c82e14d8696587397 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 18:43:55 -0700 Subject: [PATCH 16/30] real_connect_data to debug hotmail --- lib/contacts/hotmail.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index b3c41b4..930d921 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -9,6 +9,31 @@ class Hotmail < Base PWDPAD = "IfYouAreReadingThisYouHaveTooMuchFreeTime" MAX_HTTP_THREADS = 8 + def real_connect_data + data, resp, cookies, forward = get(URL) + old_url = URL + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + postdata = "PPSX=%s&PwdPad=%s&login=%s&passwd=%s&LoginOptions=2&PPFT=%s" % [ + CGI.escape(data.split("><").grep(/PPSX/).first[/=\S+$/][2..-3]), + PWDPAD[0...(PWDPAD.length-@password.length)], + CGI.escape(login), + CGI.escape(password), + CGI.escape(data.split("><").grep(/PPFT/).first[/=\S+$/][2..-3]) + ] + + form_url = data.split("><").grep(/form/).first.split[5][8..-2] + data, resp, cookies, forward = post(form_url, postdata, cookies) + + old_url = form_url + until cookies =~ /; PPAuth=/ || forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + data + end + def real_connect data, resp, cookies, forward = get(URL) old_url = URL @@ -123,4 +148,4 @@ def get_contact_list_url(index) TYPES[:hotmail] = Hotmail end -end \ No newline at end of file +end From cfa898b7a5c02f9e1e766954d996b230d5e09f7f Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 18:47:31 -0700 Subject: [PATCH 17/30] disabling connect for debugging --- lib/contacts/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/base.rb b/lib/contacts/base.rb index 3089dc2..007115a 100644 --- a/lib/contacts/base.rb +++ b/lib/contacts/base.rb @@ -18,7 +18,7 @@ def initialize(login, password, options={}) @captcha_token = options[:captcha_token] @captcha_response = options[:captcha_response] @connections = {} - connect + #connect end def connect From cbbdd50796bd4aa98a9c640638f3221d352b0c4d Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 18:58:26 -0700 Subject: [PATCH 18/30] more debugging --- lib/contacts/hotmail.rb | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 930d921..d3c6b34 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -9,7 +9,7 @@ class Hotmail < Base PWDPAD = "IfYouAreReadingThisYouHaveTooMuchFreeTime" MAX_HTTP_THREADS = 8 - def real_connect_data + def real_connect_debug data, resp, cookies, forward = get(URL) old_url = URL until forward.nil? @@ -32,7 +32,29 @@ def real_connect_data data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] end data + + if data.index("The e-mail address or password is incorrect") + raise AuthenticationError, "Username and password do not match" + elsif cookies == "" + raise ConnectionError, PROTOCOL_ERROR + end + + data, resp, cookies, forward = get("http://mail.live.com/mail", cookies) + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + + @domain = URI.parse(old_url).host + @cookies = cookies + rescue AuthenticationError => m + if @attempt == 1 + retry + else + raise m + end end + end def real_connect data, resp, cookies, forward = get(URL) From a06ec76737670e2f5382eca443acdcce6203e92d Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 19:03:22 -0700 Subject: [PATCH 19/30] whacking a superfluous "end" --- lib/contacts/hotmail.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index d3c6b34..464ea1b 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -54,8 +54,7 @@ def real_connect_debug raise m end end - end - + def real_connect data, resp, cookies, forward = get(URL) old_url = URL From 2fab02bbbfc45ffc2a5d4dc91ad5234f2dcd7592 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Wed, 13 Jul 2011 19:12:34 -0700 Subject: [PATCH 20/30] more debugging. --- lib/contacts/hotmail.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 464ea1b..626de78 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -102,6 +102,13 @@ def real_connect end end + def debug_contacts + if connected? + url = URI.parse(contact_list_url) + data, resp, cookies, forward = get( contact_list_url, @cookies ) + end + end + def contacts(options = {}) if connected? url = URI.parse(contact_list_url) From 6839ab02e24b7d5ddbd61f035d369607ee1a3825 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 11:20:22 -0700 Subject: [PATCH 21/30] updating contacts_data --- lib/contacts/aol.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 69ac22f..1db036f 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -139,9 +139,8 @@ def contacts_data "fileType" => 'csv' } - return @contacts if @contacts if connected? - data, resp, cookies, forward, old_url = get(CONTACT_LIST_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] @@ -157,7 +156,7 @@ def contacts_data postdata["user"] = input.attributes["value"] if input.attributes["name"] == "user" end - data, resp, cookies, forward, old_url = get(CONTACT_LIST_CSV_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_csv_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] From 1869c3a9064b1cacee662bbf477f5db8a90aab77 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 11:34:09 -0700 Subject: [PATCH 22/30] AOL supports 2 emails for a single contact: display both --- lib/contacts/aol.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 1db036f..4829b27 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -183,9 +183,16 @@ def parse(data, options={}) end end col_names = data.shift - @contacts = data.map do |person| - ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? - end.compact + @contacts = [] + data.each do |person| + if person[4] && !person[4].empty? && person[5] && !person[5].empty? + @contacts << ["#{person[0]} #{person[1]}", person[4]] + end + if person[5] && !person[5].empty? + @contacts << ["#{person[0]} #{person[1]}", person[5]] + end + end + @contacts end def h_to_query_string(hash) From fc9dd827495e53110bc87f4ad3096764bea784a6 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 11:51:42 -0700 Subject: [PATCH 23/30] contacts_data for yahoo --- lib/contacts/yahoo.rb | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index 7f90f0e..f79d60e 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -40,6 +40,37 @@ def real_connect(attempt_count=0) @cookies = cookies end + def contacts_data + if connected? + # first, get the addressbook site with the new crumb parameter + url = URI.parse(address_book_url) + http = open_http(url) + resp, data = http.get("#{url.path}?#{url.query}", + "Cookie" => @cookies + ) + + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + crumb = data.to_s[/dotCrumb: '(.*?)'/][13...-1] + + # now proceed with the new ".crumb" parameter to get the csv data + url = URI.parse(contact_list_url.sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) + http = open_http(url) + resp, more_data = http.get("#{url.path}?#{url.query}", + "Cookie" => @cookies, + "X-Requested-With" => "XMLHttpRequest", + "Referer" => address_book_url + ) + + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + data, more_data + end + def contacts return @contacts if @contacts if connected? @@ -113,4 +144,4 @@ def parse(data, options={}) end TYPES[:yahoo] = Yahoo -end \ No newline at end of file +end From 163ef0411a2769595259ffd1adbf66ebb444f201 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 12:00:24 -0700 Subject: [PATCH 24/30] I could've sworn implicit tuple worked. weird. --- lib/contacts/yahoo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index f79d60e..ca44824 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -68,7 +68,7 @@ def contacts_data raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - data, more_data + [data, more_data] end def contacts From 49868d6978f62951eb8a78330d407afc38597c3c Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 12:04:30 -0700 Subject: [PATCH 25/30] dropped an "end" --- lib/contacts/yahoo.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index ca44824..d868b25 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -70,7 +70,8 @@ def contacts_data [data, more_data] end - + end + def contacts return @contacts if @contacts if connected? From c7a2546c66f6fa12d341422bba01f00e8ba89c32 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 12:26:11 -0700 Subject: [PATCH 26/30] eliminating dupe returns from Yahoo, making no_connect an optional param --- lib/contacts/base.rb | 2 +- lib/contacts/yahoo.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/contacts/base.rb b/lib/contacts/base.rb index 007115a..935dae5 100644 --- a/lib/contacts/base.rb +++ b/lib/contacts/base.rb @@ -18,7 +18,7 @@ def initialize(login, password, options={}) @captcha_token = options[:captcha_token] @captcha_response = options[:captcha_response] @connections = {} - #connect + connect unless options[:no_connect] end def connect diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index d868b25..22cac0b 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -124,8 +124,8 @@ def contacts parse more_data end end - - @contacts + # eliminate dupe emails + @contacts = @contacts.uniq {|a,b| b} end end From 3990f54a726e52c7d929baed1c202a00e40bbe9e Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Thu, 14 Jul 2011 16:33:02 -0700 Subject: [PATCH 27/30] minor bit of cleanup: mostly whacking the superfluous FasterCSV --- geminstaller.yml | 2 -- ...-contacts.gemspec => joshuaknox-contacts.gemspec | 13 ++++++------- lib/contacts/base.rb | 6 +++--- lib/contacts/json_picker.rb | 5 ++--- 4 files changed, 11 insertions(+), 15 deletions(-) rename glennfu-contacts.gemspec => joshuaknox-contacts.gemspec (73%) diff --git a/geminstaller.yml b/geminstaller.yml index 9f3b0e4..26448aa 100644 --- a/geminstaller.yml +++ b/geminstaller.yml @@ -6,5 +6,3 @@ gems: version: >= 1.1.1 - name: gdata version: >= 1.1.1 - - name: fastercsv - version: >= 1.5.4 diff --git a/glennfu-contacts.gemspec b/joshuaknox-contacts.gemspec similarity index 73% rename from glennfu-contacts.gemspec rename to joshuaknox-contacts.gemspec index caa8b03..ff33141 100644 --- a/glennfu-contacts.gemspec +++ b/joshuaknox-contacts.gemspec @@ -1,15 +1,14 @@ Gem::Specification.new do |s| - s.name = "glennfu-contacts" - s.version = "1.2.6" - s.date = "2011-06-13" + s.name = "joshuaknox-contacts" + s.version = "1.2.7" + s.date = "2011-07-14" s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." - s.email = "glenn@glennfu.com" - s.homepage = "http://github.com/glennfu/contacts" + s.email = "joshuaknox@gmail.com" + s.homepage = "http://github.com/joshuaknox/contacts" s.description = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." s.has_rdoc = false - s.authors = ["Glenn Sidney", "Lucas Carlson"] + s.authors = ["Joshua Knox", "Glenn Sidney", "Lucas Carlson"] s.files = ["LICENSE", "Rakefile", "README", "examples/grab_contacts.rb", "lib/contacts.rb", "lib/contacts/base.rb", "lib/contacts/json_picker.rb", "lib/contacts/gmail.rb", "lib/contacts/aol.rb", "lib/contacts/hotmail.rb", "lib/contacts/plaxo.rb", "lib/contacts/yahoo.rb"] s.add_dependency("json", ">= 1.1.1") s.add_dependency('gdata', '>= 1.1.1') - s.add_dependency('fastercsv', '>= 1.5.4') end \ No newline at end of file diff --git a/lib/contacts/base.rb b/lib/contacts/base.rb index 935dae5..6e5b638 100644 --- a/lib/contacts/base.rb +++ b/lib/contacts/base.rb @@ -166,9 +166,9 @@ def get(url, cookies="", referer="") data = uncompress(resp, data) cookies = parse_cookies(resp.response['set-cookie'], cookies) forward = resp.response['Location'] - if (not forward.nil?) && URI.parse(forward).host.nil? - forward = url.scheme.to_s + "://" + url.host.to_s + forward - end + if (not forward.nil?) && URI.parse(forward).host.nil? + forward = url.scheme.to_s + "://" + url.host.to_s + forward + end return data, resp, cookies, forward end diff --git a/lib/contacts/json_picker.rb b/lib/contacts/json_picker.rb index 6149b62..c935f61 100644 --- a/lib/contacts/json_picker.rb +++ b/lib/contacts/json_picker.rb @@ -4,8 +4,7 @@ class Contacts def self.parse_json( string ) - if Object.const_defined?('ActiveSupport') and - ActiveSupport.const_defined?('JSON') + if Object.const_defined?('ActiveSupport') and ActiveSupport.const_defined?('JSON') ActiveSupport::JSON.decode( string ) elsif Object.const_defined?('JSON') JSON.parse( string ) @@ -13,4 +12,4 @@ def self.parse_json( string ) raise 'Contacts requires JSON or Rails (with ActiveSupport::JSON)' end end -end \ No newline at end of file +end From 703cee88c001459a2ea998ed3c56bd3e44c47770 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Sat, 16 Jul 2011 13:46:44 -0700 Subject: [PATCH 28/30] correcting the conditionals --- lib/contacts/aol.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 4829b27..df19528 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -185,7 +185,7 @@ def parse(data, options={}) col_names = data.shift @contacts = [] data.each do |person| - if person[4] && !person[4].empty? && person[5] && !person[5].empty? + if person[4] && !person[4].empty? @contacts << ["#{person[0]} #{person[1]}", person[4]] end if person[5] && !person[5].empty? From c449911251ac73cfa10158a956c831ad91857cc9 Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Mon, 18 Jul 2011 10:56:36 -0700 Subject: [PATCH 29/30] contacts_data for Plaxo --- lib/contacts/plaxo.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/contacts/plaxo.rb b/lib/contacts/plaxo.rb index e86e51a..0fa6bf0 100644 --- a/lib/contacts/plaxo.rb +++ b/lib/contacts/plaxo.rb @@ -24,6 +24,18 @@ def contacts parse data end # contacts + def contacts_data + getdata = "&authInfo.authByEmail.email=%s" % CGI.escape(login) + getdata += "&authInfo.authByEmail.password=%s" % CGI.escape(password) + data, resp, cookies, forward = get(CONTACT_LIST_URL + getdata) + + if resp.code_type != Net::HTTPOK + raise ConnectionError, PROTOCOL_ERROR + end + + data + end + private def parse(data, options={}) doc = REXML::Document.new(data) @@ -127,4 +139,4 @@ def parse(data, options={}) 3 -=end \ No newline at end of file +=end From c39e2f852e1d4741828f6938a0ef338c22f0532a Mon Sep 17 00:00:00 2001 From: joshuaknox Date: Mon, 18 Jul 2011 11:32:39 -0700 Subject: [PATCH 30/30] grab all emails for a given Plaxo contact --- lib/contacts/plaxo.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/contacts/plaxo.rb b/lib/contacts/plaxo.rb index 0fa6bf0..8fa6df5 100644 --- a/lib/contacts/plaxo.rb +++ b/lib/contacts/plaxo.rb @@ -51,14 +51,16 @@ def parse(data, options={}) elsif cont.elements['displayName'] cont.elements['displayName'].text end - email = if cont.elements['email1'] - cont.elements['email1'].text - end - if name || email - @contacts << [name, email] + i = 1 + emails = [] + while (cont.elements["email#{i}"] || cont.elements["workEmail#{i}"]) + emails << cont.elements["email#{i}"].text if cont.elements["email#{i}"] + emails << cont.elements["workEmail#{i}"].text if cont.elements["workEmail#{i}"] + i += 1 end + emails.each {|email| @contacts << [name, email] } end - @contacts + @contacts.uniq {|a,b| b} else raise ConnectionError, PROTOCOL_ERROR end