From c2f549571e2706b18987247e82fc1c4b422fe9e2 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Thu, 26 Oct 2017 14:50:22 -0400 Subject: [PATCH 1/9] initial --- lib/ghtorrent/ghtorrent.rb | 8 ++++- .../030_add_issue_comment_reactions.rb | 29 +++++++++++++++++++ lib/ghtorrent/retriever.rb | 3 +- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index 5a0643b..183b727 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -1665,7 +1665,13 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) :comment_id => comment_id, :issue_id => issue[:id], :user_id => unless user.nil? then user[:id] end, - :created_at => date(retrieved['created_at']) + :created_at => date(retrieved['created_at']). + :like => retrieved['+1'], + :dislike => retrieved['-1'], + :laugh => retrieved['laugh'], + :confused => retrieved['confused'], + :heart => retrieved['heart'], + :hooray => retrieved['hooray'] ) info "Added issue_comment #{issue_comment_str}" diff --git a/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb b/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb new file mode 100644 index 0000000..d09d3e2 --- /dev/null +++ b/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb @@ -0,0 +1,29 @@ +require 'sequel' + +require 'ghtorrent/migrations/mysql_defaults' + +Sequel.migration do + up do + puts 'Adding reaction columns to issue_comments' + alter_table :issue_comments do + add_column :like, Integer, :null => false, :default => 0 + add_column :dislike, Integer, :null => false, :default => 0 + add_column :laugh, Integer, :null => false, :default => 0 + add_column :confused, Integer, :null => false, :default => 0 + add_column :laugh, Integer, :null => false, :default => 0 + add_column :heart, Integer, :null => false, :default => 0 + add_column :hooray, Integer, :null => false, :default => 0 + end + end + + down do + puts 'Dropping reaction columns from issue_comments' + drop_column :like + drop_column :dislike + drop_column :laugh + drop_column :confused + drop_column :laugh + drop_column :heart + drop_column :hooray + end +end diff --git a/lib/ghtorrent/retriever.rb b/lib/ghtorrent/retriever.rb index b87f8fa..b94a130 100644 --- a/lib/ghtorrent/retriever.rb +++ b/lib/ghtorrent/retriever.rb @@ -522,7 +522,8 @@ def retrieve_issue_comment(owner, repo, issue_id, comment_id) 'issue_id' => issue_id, 'id' => comment_id}).first if comment.nil? - r = api_request(ghurl "repos/#{owner}/#{repo}/issues/comments/#{comment_id}") + r = api_request(ghurl("repos/#{owner}/#{repo}/issues/comments/#{comment_id}"), + media_type = 'application/vnd.github.squirrel-girl-preview') # volatile: https://developer.github.com/v3/issues/comments/#reactions-summary if r.nil? or r.empty? warn "Could not find issue_comment #{owner}/#{repo} #{issue_id}->#{comment_id}. Deleted?" From d8d243541136a716b19ee8e02a2e69752d63d239 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Fri, 27 Oct 2017 12:12:52 -0400 Subject: [PATCH 2/9] add comment reactions to retriever, persister --- lib/ghtorrent/api_client.rb | 8 ++++---- lib/ghtorrent/ghtorrent.rb | 18 ++++++++++-------- .../030_add_issue_comment_reactions.rb | 14 ++++++-------- lib/ghtorrent/retriever.rb | 4 +++- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/ghtorrent/api_client.rb b/lib/ghtorrent/api_client.rb index b64991e..61a9379 100644 --- a/lib/ghtorrent/api_client.rb +++ b/lib/ghtorrent/api_client.rb @@ -18,10 +18,10 @@ module APIClient # A paged request. Used when the result can expand to more than one # result pages. def paged_api_request(url, pages = config(:mirror_history_pages_back), - last = nil) - + last = nil, media_type = '') + info "media type: #{media_type.inspect}" url = ensure_max_per_page(url) - data = api_request_raw(url) + data = api_request_raw(url, media_type) return [] if data.nil? @@ -39,7 +39,7 @@ def paged_api_request(url, pages = config(:mirror_history_pages_back), if links['next'].nil? parse_request_result(data) else - parse_request_result(data) | paged_api_request(links['next'], pages, last) + parse_request_result(data) | paged_api_request(links['next'], pages, last, media_type) end else parse_request_result(data) diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index 183b727..edab31b 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -37,6 +37,7 @@ def db return @db unless @db.nil? Sequel.single_threaded = true + Sequel.split_symbols = true @db = Sequel.connect(config(:sql_url), :encoding => 'utf8') #@db.loggers << Logger.new(STDOUT) if @db.tables.empty? @@ -1660,18 +1661,19 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) end user = ensure_user(retrieved['user']['login'], false, false) - + reactions = retrieved['reactions'] + info "Reactions: #{reactions.inspect}" db[:issue_comments].insert( :comment_id => comment_id, :issue_id => issue[:id], :user_id => unless user.nil? then user[:id] end, - :created_at => date(retrieved['created_at']). - :like => retrieved['+1'], - :dislike => retrieved['-1'], - :laugh => retrieved['laugh'], - :confused => retrieved['confused'], - :heart => retrieved['heart'], - :hooray => retrieved['hooray'] + :created_at => date(retrieved['created_at']), + :like => reactions['+1'], + :dislike => reactions['-1'], + :laugh => reactions['laugh'], + :confused => reactions['confused'], + :heart => reactions['heart'], + :hooray => reactions['hooray'] ) info "Added issue_comment #{issue_comment_str}" diff --git a/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb b/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb index d09d3e2..7d71605 100644 --- a/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb +++ b/lib/ghtorrent/migrations/030_add_issue_comment_reactions.rb @@ -6,13 +6,12 @@ up do puts 'Adding reaction columns to issue_comments' alter_table :issue_comments do - add_column :like, Integer, :null => false, :default => 0 - add_column :dislike, Integer, :null => false, :default => 0 - add_column :laugh, Integer, :null => false, :default => 0 - add_column :confused, Integer, :null => false, :default => 0 - add_column :laugh, Integer, :null => false, :default => 0 - add_column :heart, Integer, :null => false, :default => 0 - add_column :hooray, Integer, :null => false, :default => 0 + add_column :like, Integer + add_column :dislike, Integer + add_column :laugh, Integer + add_column :confused, Integer + add_column :heart, Integer + add_column :hooray, Integer end end @@ -22,7 +21,6 @@ drop_column :dislike drop_column :laugh drop_column :confused - drop_column :laugh drop_column :heart drop_column :hooray end diff --git a/lib/ghtorrent/retriever.rb b/lib/ghtorrent/retriever.rb index b94a130..71b851b 100644 --- a/lib/ghtorrent/retriever.rb +++ b/lib/ghtorrent/retriever.rb @@ -496,7 +496,9 @@ def retrieve_issue_event(owner, repo, issue_id, event_id) def retrieve_issue_comments(owner, repo, issue_id) url = ghurl "repos/#{owner}/#{repo}/issues/#{issue_id}/comments" - retrieved_comments = paged_api_request url + + retrieved_comments = paged_api_request(url, config(:mirror_history_pages_back), + nil, 'application/vnd.github.squirrel-girl-preview') comments = retrieved_comments.each { |x| x['owner'] = owner From c0d20155661ece919b83d1bd11341afcee35dfeb Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Fri, 27 Oct 2017 12:14:44 -0400 Subject: [PATCH 3/9] formatting --- lib/ghtorrent/api_client.rb | 2 +- lib/ghtorrent/ghtorrent.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ghtorrent/api_client.rb b/lib/ghtorrent/api_client.rb index 61a9379..a399d17 100644 --- a/lib/ghtorrent/api_client.rb +++ b/lib/ghtorrent/api_client.rb @@ -19,7 +19,7 @@ module APIClient # result pages. def paged_api_request(url, pages = config(:mirror_history_pages_back), last = nil, media_type = '') - info "media type: #{media_type.inspect}" + url = ensure_max_per_page(url) data = api_request_raw(url, media_type) diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index edab31b..ed66253 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -1662,7 +1662,7 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) user = ensure_user(retrieved['user']['login'], false, false) reactions = retrieved['reactions'] - info "Reactions: #{reactions.inspect}" + db[:issue_comments].insert( :comment_id => comment_id, :issue_id => issue[:id], From e638951c6640b36b600be1f8b63dfb12d13ccca0 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Fri, 27 Oct 2017 14:51:52 -0400 Subject: [PATCH 4/9] more --- lib/ghtorrent/ghtorrent.rb | 20 ++++++++++++++++++++ lib/ghtorrent/migrations/011_add_issues.rb | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index ed66253..8869246 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -1681,6 +1681,26 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) :comment_id => comment_id) else debug "Issue comment #{issue_comment_str} exists" + if ! (curcomment[:like] == reactions['+1'] or + curcomment[:dislike] == reactions['-1'] or + curcomment[:laugh] == reactions['laugh'] or + curcomment[:confused] == reactions['confused'] or + curcomment[:heart] == reactions['heart'] or + curcomment[:hooray] == reactions['hooray'] + ) + info "Updating issue comment reactions..." + + reactions = retrieve_issue_comment(owner, repo, issue_id, comment_id)['reactions'] + db[:issue_comments].filter(:issue_id => issue[:id], + :comment_id => comment_id).update(:like => reactions['+1'], + :dislike => reactions['-1'], + :laugh => reactions['laugh'], + :confused => reactions['confused'], + :heart => reactions['heart'], + :hooray => reactions['hooray']) + else + info "Comment reactions current" + end curcomment end end diff --git a/lib/ghtorrent/migrations/011_add_issues.rb b/lib/ghtorrent/migrations/011_add_issues.rb index 136dd20..d3a8a95 100644 --- a/lib/ghtorrent/migrations/011_add_issues.rb +++ b/lib/ghtorrent/migrations/011_add_issues.rb @@ -71,4 +71,4 @@ drop_table :issue_events drop_table :issues end -end \ No newline at end of file +end From 14b6994ce4cc769af8e5604e5974716569d26589 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Mon, 30 Oct 2017 13:48:33 -0400 Subject: [PATCH 5/9] safely apply update logic --- lib/ghtorrent/ghtorrent.rb | 16 ++++++++-------- lib/ghtorrent/migrations/011_add_issues.rb | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index 1a4b73f..494ee92 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -1693,16 +1693,16 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) :comment_id => comment_id) else debug "Issue comment #{issue_comment_str} exists" - if ! (curcomment[:like] == reactions['+1'] or - curcomment[:dislike] == reactions['-1'] or - curcomment[:laugh] == reactions['laugh'] or - curcomment[:confused] == reactions['confused'] or - curcomment[:heart] == reactions['heart'] or - curcomment[:hooray] == reactions['hooray'] - ) + if ! (curcomment[:like] == reactions['+1']) or + ! (curcomment[:dislike] == reactions['-1']) or + ! (curcomment[:laugh] == reactions['laugh']) or + ! (curcomment[:confused] == reactions['confused']) or + ! (curcomment[:heart] == reactions['heart']) or + ! (curcomment[:hooray] == reactions['hooray']) info "Updating issue comment reactions..." - reactions = retrieve_issue_comment(owner, repo, issue_id, comment_id)['reactions'] + retrieved = retrieve_issue_comment(owner, repo, issue_id, comment_id) + reactions = retrieved['reactions'] db[:issue_comments].filter(:issue_id => issue[:id], :comment_id => comment_id).update(:like => reactions['+1'], :dislike => reactions['-1'], diff --git a/lib/ghtorrent/migrations/011_add_issues.rb b/lib/ghtorrent/migrations/011_add_issues.rb index d3a8a95..258492b 100644 --- a/lib/ghtorrent/migrations/011_add_issues.rb +++ b/lib/ghtorrent/migrations/011_add_issues.rb @@ -27,7 +27,6 @@ String :action_specific, :null => true, :size => 50 DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP String :ext_ref_id, :null => false, :size => 24, :default => "0" - check(:action=>%w[closed reopened subscribed merged referenced mentioned assigned]) primary_key [:event_id, :issue_id], :name=>:issue_events_pk end From dc3e4bd7a6bf16456166d5ec6bf081a8cc1a2a32 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Mon, 30 Oct 2017 13:54:53 -0400 Subject: [PATCH 6/9] do not remove constraint --- lib/ghtorrent/migrations/011_add_issues.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ghtorrent/migrations/011_add_issues.rb b/lib/ghtorrent/migrations/011_add_issues.rb index 258492b..d3a8a95 100644 --- a/lib/ghtorrent/migrations/011_add_issues.rb +++ b/lib/ghtorrent/migrations/011_add_issues.rb @@ -27,6 +27,7 @@ String :action_specific, :null => true, :size => 50 DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP String :ext_ref_id, :null => false, :size => 24, :default => "0" + check(:action=>%w[closed reopened subscribed merged referenced mentioned assigned]) primary_key [:event_id, :issue_id], :name=>:issue_events_pk end From 0029c12ac1c2e48d5ed06f77e33b861a88c32088 Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Fri, 3 Nov 2017 15:24:45 -0400 Subject: [PATCH 7/9] fix bug --- lib/ghtorrent/commands/full_repo_retriever.rb | 2 +- lib/ghtorrent/ghtorrent.rb | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/ghtorrent/commands/full_repo_retriever.rb b/lib/ghtorrent/commands/full_repo_retriever.rb index 4e6ba58..2f5cf38 100644 --- a/lib/ghtorrent/commands/full_repo_retriever.rb +++ b/lib/ghtorrent/commands/full_repo_retriever.rb @@ -114,7 +114,7 @@ def retrieve_full_repo(owner, repo) send(event['type'], event) info "Success processing event. Type: #{event['type']}, ID: #{event['id']}" rescue StandardError => e - warn "Error processing event. Type: #{event['type']}, ID: #{event['id']}" + warn "Error processing event. Type: #{event['type']}, ID: #{event['id']}.\nError: #{$!}\n#{e.backtrace.join("\n")}." end end end diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index 494ee92..e1e2c17 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -1663,15 +1663,16 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) curcomment = db[:issue_comments].first(:issue_id => issue[:id], :comment_id => comment_id) - if curcomment.nil? + retrieved = retrieve_issue_comment(owner, repo, issue_id, comment_id) - retrieved = retrieve_issue_comment(owner, repo, issue_id, comment_id) + if retrieved.nil? + warn "Could not retrieve issue_comment #{issue_comment_str}" + return + end - if retrieved.nil? - warn "Could not retrieve issue_comment #{issue_comment_str}" - return - end + reactions = retrieved['reactions'] + if curcomment.nil? user = ensure_user(retrieved['user']['login'], false, false) reactions = retrieved['reactions'] @@ -1692,7 +1693,6 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) db[:issue_comments].first(:issue_id => issue[:id], :comment_id => comment_id) else - debug "Issue comment #{issue_comment_str} exists" if ! (curcomment[:like] == reactions['+1']) or ! (curcomment[:dislike] == reactions['-1']) or ! (curcomment[:laugh] == reactions['laugh']) or From 7982fa72f54eeb3fde6dfc5e3726a76cc4853dff Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Fri, 3 Nov 2017 15:31:21 -0400 Subject: [PATCH 8/9] fix sql bug --- lib/ghtorrent/event_processing.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ghtorrent/event_processing.rb b/lib/ghtorrent/event_processing.rb index 5dde64d..518a7aa 100644 --- a/lib/ghtorrent/event_processing.rb +++ b/lib/ghtorrent/event_processing.rb @@ -44,9 +44,9 @@ def PushEvent(data) next if push_commits.include? url[7] sha_not_exist = ght.db.from(:commits, :project_commits, :projects, :users).\ - where(Sequel.qualify('projects', 'id') => Sequel.qualify('project_commits', 'project_id)')).\ - where(Sequel.qualify('commits', 'id') => Sequel.qualify('project_commits', 'commit_id)')).\ - where(Sequel.qualify('projects', 'owner_id') => Sequel.qualify('users', 'id)')).\ + where(Sequel.qualify('projects', 'id') => Sequel.qualify('project_commits', 'project_id')).\ + where(Sequel.qualify('commits', 'id') => Sequel.qualify('project_commits', 'commit_id')).\ + where(Sequel.qualify('projects', 'owner_id') => Sequel.qualify('users', 'id')).\ where(Sequel.qualify('projects', 'name') => url[5]).\ where(Sequel.qualify('users', 'login') => url[4]).\ where(Sequel.qualify('commits', 'sha') => url[7]).all.empty? From 028068240835264e3b0e053f2eef959041a93cea Mon Sep 17 00:00:00 2001 From: Andrew Colello Date: Mon, 6 Nov 2017 10:34:42 -0500 Subject: [PATCH 9/9] remove split symbols bool --- lib/ghtorrent/ghtorrent.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index e1e2c17..19fca5a 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -37,7 +37,6 @@ def db return @db unless @db.nil? Sequel.single_threaded = true - Sequel.split_symbols = true @db = Sequel.connect(config(:sql_url), :encoding => 'utf8') #@db.loggers << Logger.new(STDOUT) if @db.tables.empty?