From edbc83380dca98531e6bdaf9440d2edba362aa95 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 15:35:07 +0200 Subject: [PATCH 001/293] fix typo of closing span tag --- views/index.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/index.erb b/views/index.erb index b56c5bf..942f027 100644 --- a/views/index.erb +++ b/views/index.erb @@ -5,7 +5,7 @@
- <%= '↓ ' if params[:sort_by].nil? || params[:sort_by] == 'latest' %>most recent · + <%= '↓ ' if params[:sort_by].nil? || params[:sort_by] == 'latest' %>most recent · <%= '↓ ' if params[:sort_by] == 'frequent' %>most frequent
From be40f0161df833d0ee5857492f3d149207684fe2 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 15:38:04 +0200 Subject: [PATCH 002/293] add newline to end of document --- views/new_exceptions.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/new_exceptions.erb b/views/new_exceptions.erb index 57a1bda..89d7bfd 100644 --- a/views/new_exceptions.erb +++ b/views/new_exceptions.erb @@ -21,4 +21,4 @@ <% end %> - \ No newline at end of file + From 132e142377269c86a886e6d18005ec7383378856 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 15:39:35 +0200 Subject: [PATCH 003/293] same naming for occurrences_count in project --- public/stylesheets/exceptionist.css | 2 +- views/_exceptions.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index fd41f3b..abdc4f9 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -74,7 +74,7 @@ dd { text-align: right; } -span.occurrence_count { +span.occurrences_count { float: right; background-color: #eee; font-size: 1.2em; diff --git a/views/_exceptions.erb b/views/_exceptions.erb index 76890a5..8555607 100644 --- a/views/_exceptions.erb +++ b/views/_exceptions.erb @@ -8,7 +8,7 @@ <%= escape_html(truncate(uber_exc.url, 80)) %>
- # <%= uber_exc.occurrences_count %> + # <%= uber_exc.occurrences_count %>
<% end %> From 0fd9c0d81c4c0e3ab3216af9c78318a7e5c1ad23 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 15:47:49 +0200 Subject: [PATCH 004/293] don't ignore Gemfile.lock --- .gitignore | 1 - Gemfile.lock | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 23deb6d..9fe02fd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,5 @@ webrat.log *.json vendor/bundle .bundle -Gemfile.lock .idea diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..1c90ff6 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,84 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (4.1.5) + i18n (~> 0.6, >= 0.6.9) + json (~> 1.7, >= 1.7.7) + minitest (~> 5.1) + thread_safe (~> 0.1) + tzinfo (~> 1.1) + ansi (1.4.3) + bson (1.10.2) + bson_ext (1.10.2) + bson (~> 1.10.2) + elasticsearch (1.0.4) + elasticsearch-api (= 1.0.4) + elasticsearch-transport (= 1.0.4) + elasticsearch-api (1.0.4) + multi_json + elasticsearch-extensions (0.0.15) + ansi + ruby-prof + elasticsearch-transport (1.0.4) + faraday + multi_json + faraday (0.9.0) + multipart-post (>= 1.2, < 3) + hashie (2.1.2) + i18n (0.6.11) + json (1.8.1) + kgio (2.9.2) + minitest (5.4.0) + mongo (1.10.2) + bson (= 1.10.2) + multi_json (1.10.1) + multipart-post (2.0.0) + nokogiri (1.5.11) + patron (0.4.18) + rack (1.5.2) + rack-protection (1.5.3) + rack + rack-test (0.6.2) + rack (>= 1.0) + raindrops (0.13.0) + rake (10.3.2) + ruby-prof (0.15.1) + shotgun (0.9) + rack (>= 1.0) + sinatra (1.4.5) + rack (~> 1.4) + rack-protection (~> 1.4) + tilt (~> 1.3, >= 1.3.4) + thread_safe (0.3.4) + tilt (1.4.1) + tzinfo (1.2.2) + thread_safe (~> 0.1) + unicorn (4.8.3) + kgio (~> 2.6) + rack + raindrops (~> 0.7) + webrat (0.7.3) + nokogiri (>= 1.2.0) + rack (>= 1.0) + rack-test (>= 0.5.3) + yajl-ruby (1.2.1) + +PLATFORMS + ruby + +DEPENDENCIES + activesupport + bson_ext + elasticsearch + elasticsearch-extensions + hashie + mongo + nokogiri (~> 1.5.10) + patron + rack-test + rake + shotgun + sinatra + unicorn + webrat + yajl-ruby From 97969d0427bbc280043e517635fcb12f724a0c81 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 15:51:19 +0200 Subject: [PATCH 005/293] new gems for elasticsearch db --- Gemfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index f88b840..4b9e070 100644 --- a/Gemfile +++ b/Gemfile @@ -10,9 +10,13 @@ gem 'sinatra' gem 'nokogiri', '~> 1.5.10' gem 'yajl-ruby' gem 'activesupport' +gem 'elasticsearch' +gem 'patron' +gem 'hashie' group :development do gem 'rack-test' gem 'webrat' gem 'shotgun' + gem 'elasticsearch-extensions' end From a079c534af246ba10dfb37e8346ff142e92d0c87 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 29 Aug 2014 16:00:37 +0200 Subject: [PATCH 006/293] use elasticsearch instead of mongodb --- Rakefile | 12 - lib/app.rb | 7 +- lib/es_client.rb | 109 +++++++++ lib/exceptionist.rb | 17 +- lib/models/occurrence.rb | 38 +-- lib/models/project.rb | 4 +- lib/models/uber_exception.rb | 73 +++--- lib/tools.rb | 24 -- test/api_test.rb | 73 ++++-- test/elastic_search_helper.rb | 36 +++ .../exception_with_hash_in_params.xml | 4 +- test/fixtures/full_exception.xml | 22 +- test/integration_test.rb | 72 +++--- test/mongod-test.conf | 3 - test/occurrence_test.rb | 217 ++++++++++++------ test/project_test.rb | 32 +++ test/test_helper.rb | 49 ++-- test/uber_exception_test.rb | 175 ++++++++++++-- 18 files changed, 693 insertions(+), 274 deletions(-) create mode 100644 lib/es_client.rb delete mode 100644 lib/tools.rb create mode 100644 test/elastic_search_helper.rb delete mode 100644 test/mongod-test.conf create mode 100644 test/project_test.rb diff --git a/Rakefile b/Rakefile index 0173426..b95533e 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,4 @@ $LOAD_PATH.unshift 'lib' -require 'tools' task :default => :test @@ -8,14 +7,3 @@ Rake::TestTask.new do |test| test.libs << "test" test.test_files = FileList['test/**/*_test.rb'] end - -desc "Remove a single exception with all occurrences completely" -task :remove_exception do - uber_key = ENV['KEY'] - Exceptionist::Remover.run(uber_key) -end - -desc "Create MongoDB indexes" -task :create_indexes do - Exceptionist::IndexCreator.run -end diff --git a/lib/app.rb b/lib/app.rb index 425dfe0..d05b50c 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -5,6 +5,7 @@ require 'stringio' require 'pp' + class ExceptionistApp < Sinatra::Base dir = File.join(File.dirname(__FILE__), '..') set :views, "#{dir}/views" @@ -44,7 +45,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/river' do @current_project = Project.new(params[:project]) - @occurrences = Occurrence.find_all(@current_project.name) + @occurrences = Occurrence.find_all_by_name(@current_project.name) @title = "Latest Occurrences for #{@current_project.name}" erb :river @@ -78,7 +79,7 @@ class ExceptionistApp < Sinatra::Base @occurrence = @uber_exception.current_occurrence(@occurrence_position) if @occurrence.nil? - @uber_exception.update_occurrence_count + @uber_exception.update_occurrences_count redirect request.url end @@ -106,8 +107,8 @@ class ExceptionistApp < Sinatra::Base post '/notifier_api/v2/notices/?' do occurrence = Occurrence.from_xml(params[:data] || request.body.read) - project = Project.find_by_key(occurrence.api_key) + if project occurrence.project_name = project.name occurrence.save diff --git a/lib/es_client.rb b/lib/es_client.rb new file mode 100644 index 0000000..d4df8b9 --- /dev/null +++ b/lib/es_client.rb @@ -0,0 +1,109 @@ +require 'elasticsearch' + +class ESClient + attr_accessor :es + INDEX = 'exceptionist' + TYPE_EXCEPTIONS = 'exceptions' + TYPE_OCCURRENCES = 'occurrences' + + def initialize(endpoint) + @es = Elasticsearch::Client.new(host: endpoint) + end + + def search_occurrences(filters, sort={}, from=0, size=50) + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) + hash = Hashie::Mash.new(response) + hash.hits.hits.map { |doc| create_occurrence(doc) } + end + + def search_exceptions(filters, sort={}, from=0, size=50) + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: TYPE_EXCEPTIONS, body: query) + hash = Hashie::Mash.new(response) + hash.hits.hits.map { |doc| create_exception(doc) } + end + + def search_aggs(filters, aggs) + query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs } } } } + response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) + hash = Hashie::Mash.new(response) + hash.aggregations.exceptions.buckets + end + + def search_ids(type, ids) + response = @es.search(index: INDEX, type: type, body: { query: { ids: { values: ids } } } ) + hash = Hashie::Mash.new(response) + hash.hits.hits.map { |doc| create_exception(doc) } + end + + def index(type, body) + response = @es.index(index: INDEX, type: type, body: body) + Hashie::Mash.new(response) + end + + def update(type, id, body) + @es.update(index: INDEX, type: type, id: id, body: body) + end + + def count(type, terms) + terms = [terms] if terms.class == Hash + query = { query: { filtered: { filter: { bool: { must: terms } } } } } + + @es.count(index: INDEX, type: type, body: query)['count'] + end + + def delete_by_query(query) + @es.delete_by_query( index: INDEX, body: { query: query } ) + end + + def delete(type, id) + @es.delete(index: INDEX, type: type, id: id) + end + + def delete_indices(index) + @es.indices.delete(index: index) + end + + def create_indices(index, query) + @es.indices.create(index: index, body: query) + end + + def get_exception(id) + response = @es.get(index: INDEX, type: TYPE_EXCEPTIONS, id: id) + create_exception(response) + end + + def refresh + @es.indices.refresh + end + + private + def create_occurrence(attributes) + return nil unless attributes + + attributes.merge!(attributes['_source']).delete('_source') + Occurrence.new(attributes) + end + + def create_exception(attributes) + attributes.merge!(attributes['_source']).delete('_source') + UberException.new(attributes) + end + + def create_search_query(terms, sort, from, size) + sort = wrap_sort(sort) unless sort.empty? + return { sort: sort, from: from, size: size } if terms.empty? + { query: { filtered: { filter: { bool: { must: terms } } } }, sort: sort, from: from, size: size} + end + + def wrap_sort(sort) + sort = [sort] if sort.class == Hash + sort.each { | field | add_ignore_unmapped(field) } + end + + def add_ignore_unmapped(hash) + hash.each { | field, ordering | ordering[:ignore_unmapped] = true } + end + +end diff --git a/lib/exceptionist.rb b/lib/exceptionist.rb index 0067f8e..682e1f5 100644 --- a/lib/exceptionist.rb +++ b/lib/exceptionist.rb @@ -1,10 +1,19 @@ +require 'hashie' +require 'multi_json' +require 'faraday' +require 'elasticsearch' +require 'elasticsearch/api' +require 'es_client' + module Exceptionist - def self.mongo - @mongo ||= Mongo::Connection.new(@host, @port).db('exceptionist') + attr_accessor :esclient, :endpoint + + def self.esclient + @esclient ||= ESClient.new(@endpoint) end - def self.mongo=(server) - @host, @port = server.split(':') + def self.endpoint=(endpoint) + @endpoint = endpoint end def self.config diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 61a0d51..99a8d0d 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -2,12 +2,13 @@ class Occurrence attr_accessor :url, :controller_name, :action_name, :exception_class, :exception_message, :exception_backtrace, :parameters, :session, :cgi_data, :environment, - :project_name, :occurred_at, :occurred_at_day, :'_id', :uber_key, :api_key + :project_name, :occurred_at, :occurred_at_day, :_id, :uber_key, :api_key, + :_score, :sort def initialize(attributes={}) attributes.each do |key, value| - send("#{key}=", value) + send("#{key}=", value) unless ["_index", "_type"].include?(key) end self.occurred_at ||= attributes['occurred_at'] || Time.now @@ -56,27 +57,29 @@ def uber_exception end def self.delete_all_for(uber_key) - Exceptionist.mongo['occurrences'].remove({:uber_key => uber_key}, :w => 1) + Exceptionist.esclient.delete_by_query({ term: { uber_key: uber_key } }) end def self.find_first_for(uber_key) - new(Exceptionist.mongo['occurrences'].find({:uber_key => uber_key}, :sort => [:occurred_at, :asc], :limit => 1).first) + occurrences = Exceptionist.esclient.search_occurrences( { term: { uber_key: uber_key } }, { occurred_at: { order: 'asc' } }, from: 0, size: 1 ) + occurrences.first end def self.find_last_for(uber_key) - new(Exceptionist.mongo['occurrences'].find({:uber_key => uber_key}, :sort => [:occurred_at, :desc], :limit => 1).first) + occurrences = Exceptionist.esclient.search_occurrences( { term: { uber_key: uber_key } }, { occurred_at: { order: 'desc' } }, from: 0, size: 1 ) + occurrences.first end def self.count_all_on(project, day) - Exceptionist.mongo['occurrences'].find({:project_name => project, :occurred_at_day => day.strftime('%Y-%m-%d')}).count + Exceptionist.esclient.count('occurrences', [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) end - def self.find_all(project=nil, limit=50) - find_options = {} - find_options[:project_name] = project if project + def self.find_all(size=50) + Exceptionist.esclient.search_occurrences( {}, { occurred_at: { order: 'desc' } }, 0, size ) + end - occurrences = Exceptionist.mongo['occurrences'].find(find_options, :sort => [:occurred_at, :desc], :limit => limit) - occurrences.map { |doc| new(doc) } + def self.find_all_by_name(project, limit=50) + Exceptionist.esclient.search_occurrences( { term: { project_name: project } }, { occurred_at: { order: 'desc' } }, from: 0, size: limit ) end # @@ -84,8 +87,8 @@ def self.find_all(project=nil, limit=50) # def save - Exceptionist.mongo['occurrences'].insert(to_hash) - + occurrence = Exceptionist.esclient.index('occurrences', to_hash) + @_id = occurrence._id self end @@ -93,6 +96,13 @@ def self.create(attributes = {}) new(attributes).save end + def self.es_create(attributes) + return nil unless attributes + + attributes.merge!(attributes['_source']).delete('_source') + Occurrence.new(attributes) + end + def to_hash { :exception_message => exception_message, :session => session, @@ -100,7 +110,7 @@ def to_hash :parameters => parameters, :cgi_data => cgi_data, :url => url, - :occurred_at => occurred_at, + :occurred_at => occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), :occurred_at_day => occurred_at.strftime('%Y-%m-%d'), :exception_backtrace => exception_backtrace, :controller_name => controller_name, diff --git a/lib/models/project.rb b/lib/models/project.rb index 3672d96..8d3de74 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -26,7 +26,7 @@ def self.last_n_days(days) end def occurrence_count_on(date) - Exceptionist.mongo['occurrences'].find({:project_name => name, :occurred_at_day => date.strftime('%Y-%m-%d')}).count + Occurrence.count_all_on(name, date) end def latest_exceptions(start, limit = 25) @@ -34,7 +34,7 @@ def latest_exceptions(start, limit = 25) end def most_frequest_exceptions(start, limit = 25) - UberException.find_all_sorted_by_occurrence_count(name, start, limit) + UberException.find_all_sorted_by_occurrences_count(name, start, limit) end def new_exceptions_on(day) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 8f9abd5..6fa3179 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,53 +1,51 @@ class UberException - attr_accessor :id, :project_name, :occurrences_count + attr_accessor :id, :project_name, :occurrences_count, :closed def initialize(attributes) @id = attributes['_id'] @project_name = attributes['project_name'] - @occurrences_count = attributes['occurrence_count'] + @occurrences_count = attributes['occurrences_count'] + end + + def self.create_es(attributes) + attributes.merge!(attributes['_source']).delete('_source') + UberException.new(attributes) end def self.count_all(project) - Exceptionist.mongo['exceptions'].find({:project_name => project, :closed => false}).count + Exceptionist.esclient.count('exceptions', { term: { project_name: project } } ) end def self.find(uber_key) - new(Exceptionist.mongo['exceptions'].find_one({:_id => uber_key})) + Exceptionist.esclient.get_exception(uber_key) end def self.find_all(project) - uber_exceptions = Exceptionist.mongo['exceptions'].find({:project_name => project, :closed => false}) - uber_exceptions.map { |doc| new(doc) } + Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ]) end def self.find_all_sorted_by_time(project, start, limit) - uber_exceptions = Exceptionist.mongo['exceptions'].find({:project_name => project, :closed => false}, :skip => start, :limit => limit, :sort => [:occurred_at, :desc]) - uber_exceptions.map { |doc| new(doc) } + Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { occurred_at: { order: 'desc'} }, start, limit) end - def self.find_all_sorted_by_occurrence_count(project, start, limit) - uber_exceptions = Exceptionist.mongo['exceptions'].find({:project_name => project, :closed => false}, :skip => start, :limit => limit, :sort => [:occurrence_count, :desc]) - uber_exceptions.map { |doc| new(doc) } + def self.find_all_sorted_by_occurrences_count(project, start, limit) + Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { occurrences_count: { order: 'desc' } }, start, limit ) end def self.find_new_on(project, day) next_day = day + 86400 - uber_keys = Exceptionist.mongo['occurrences'].distinct(:uber_key, {:occurred_at_day => day.strftime('%Y-%m-%d')}) - uber_exceptions = Exceptionist.mongo['exceptions'].find({:_id => {'$in' => uber_keys}}).map { |doc| new(doc) } + + buckets = Exceptionist.esclient.search_aggs({ term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, 'uber_key' ) + uber_exceptions = Exceptionist.esclient.search_ids('exceptions', buckets.map { |occ| occ['key'] } ) uber_exceptions.select { |uber_exp| uber_exp.first_occurred_at >= day && uber_exp.first_occurred_at < next_day } end + + def self.occurred(occurrence) - # upsert the UberException - Exceptionist.mongo['exceptions'].update( - {:_id => occurrence.uber_key}, - { - "$set" => {:project_name => occurrence.project_name, :occurred_at => occurrence.occurred_at, :closed => false}, - "$inc" => {:occurrence_count => 1} - }, - :upsert => true, :w => 1 - ) + Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1', upsert: + { project_name: occurrence.project_name, occurred_at: occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), closed: false, occurrences_count: 1 } }) # return the UberException new('_id' => occurrence.uber_key) @@ -57,9 +55,10 @@ def self.forget_old_exceptions(project, days) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.mongo['exceptions'].find({:project_name => project, :occurred_at => {'$lt' => since_date}}) - uber_exceptions.each do |doc| - UberException.new(doc).forget! + uber_exceptions = Exceptionist.esclient.search_exceptions( [ { term: { project_name: project } }, range: { occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + + uber_exceptions.each do |exception| + exception.forget! deleted += 1 end @@ -68,11 +67,12 @@ def self.forget_old_exceptions(project, days) def forget! Occurrence.delete_all_for(id) - Exceptionist.mongo['exceptions'].remove({:_id => id}, :w => 1) + + Exceptionist.esclient.delete('exceptions', id) end def close! - Exceptionist.mongo['exceptions'].update({:_id => id}, {'$set' => {'closed' => true}}) + Exceptionist.esclient.update('exceptions', @id, { doc: { closed: true } }) end def last_occurrence @@ -84,16 +84,19 @@ def first_occurrence end def current_occurrence(position) - if occurrence = Exceptionist.mongo['occurrences'].find({:uber_key => id}, :sort => [:occurred_at, :asc], :skip => position - 1, :limit => 1).first - Occurrence.new(occurrence) + return nil if position < 1 + + response = Exceptionist.esclient.search_occurrences({ term: { uber_key: id } }, { occurred_at: { order: 'asc'} }, position - 1, 1) + if response.any? + response.first else nil end end - def update_occurrence_count - @occurrences_count = Exceptionist.mongo['occurrences'].find({:uber_key => id}).count - Exceptionist.mongo['exceptions'].update({:_id => id}, {'$set' => {'occurrence_count' => @occurrences_count}}) + def update_occurrences_count + @occurrences_count = Exceptionist.esclient.count('occurrences', { term: { uber_key: id } }) + Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: @occurrences_count } }) end def first_occurred_at @@ -112,12 +115,12 @@ def url last_occurrence.url end - def occurrence_count_on(date) - Exceptionist.mongo['occurrences'].find({:uber_key => id, :occurred_at_day => date.strftime('%Y-%m-%d')}).count + def occurrences_count_on(date) + Exceptionist.esclient.count('occurrences', [ { term: { uber_key: id } }, term: { occurred_at_day: date.strftime('%Y-%m-%d') } ]) end def last_thirty_days - Project.last_n_days(30).map { |day| [day, occurrence_count_on(day)] } + Project.last_n_days(30).map { |day| [day, occurrences_count_on(day)] } end def ==(other) diff --git a/lib/tools.rb b/lib/tools.rb deleted file mode 100644 index a7a5b2d..0000000 --- a/lib/tools.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'boot' -require './config' - -module Exceptionist - class Remover - def self.run(uber_key) - UberException.find(uber_key).forget! - end - end - - class IndexCreator - def self.run - Exceptionist.mongo['exceptions'].ensure_index(:project_name) - - Exceptionist.mongo['occurrences'].ensure_index(:uber_key) - Exceptionist.mongo['occurrences'].ensure_index([[:project_name, Mongo::ASCENDING], [:occurred_at_day, Mongo::ASCENDING]]) - Exceptionist.mongo['occurrences'].ensure_index([[:uber_key, Mongo::ASCENDING], [:occurred_at, Mongo::ASCENDING]]) - Exceptionist.mongo['occurrences'].ensure_index([[:uber_key, Mongo::ASCENDING], [:occurred_at_day, Mongo::ASCENDING]]) - - # River view - Exceptionist.mongo['occurrences'].ensure_index([[:occurred_at, Mongo::DESCENDING]]) - end - end -end diff --git a/test/api_test.rb b/test/api_test.rb index b367a80..38c7571 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -2,41 +2,42 @@ require 'rack/test' -class ApiTest < Minitest::Test +class ApiTest < AbstractTest include Rack::Test::Methods def app ExceptionistApp end - def setup - @project = 'ExampleProject' - clear_collections - end - def test_create_the_first_UberException - assert_equal [], UberException.find_all(@project) + assert_equal [], UberException.find_all('ExampleProject') post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? - uber_exceptions = UberException.find_all(@project) - assert_equal 1, uber_exceptions.count - assert_equal 1, uber_exceptions.first.occurrences_count + Exceptionist.esclient.refresh + + exce = UberException.find_all('ExampleProject') + assert_equal 1, exce.count + assert_equal 1, exce.first.occurrences_count end def test_add_occurrences_if_it_is_the_same_exception post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? - assert_equal 1, UberException.find_all(@project).count + Exceptionist.esclient.refresh + + assert_equal 1, UberException.find_all('ExampleProject').count post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? - uber_exceptions = UberException.find_all(@project) - assert_equal 1, uber_exceptions.count - assert_equal 2, uber_exceptions.first.occurrences_count + Exceptionist.esclient.refresh + + exce = UberException.find_all('ExampleProject') + assert_equal 1, exce.count + assert_equal 2, exce.first.occurrences_count end def test_check_if_api_key_is_valid @@ -44,6 +45,48 @@ def test_check_if_api_key_is_valid assert_equal 'Invalid API Key', last_response.body assert_equal 401, last_response.status - assert_equal [], UberException.find_all(@project) + + Exceptionist.esclient.refresh + + assert_equal [], UberException.find_all('ExampleProject') + end + + def test_api_full_exception + assert_equal [], UberException.find_all('ExampleProject') + + post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/full_exception.xml') + assert last_response.ok? + + Exceptionist.esclient.refresh + + exce = UberException.find_all('ExampleProject') + assert_equal 1, exce.count + assert_equal 1, exce.first.occurrences_count + end + + def test_api_unauth_exception + assert_equal [], UberException.find_all('ExampleProject') + + post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception_with_hash_in_params.xml') + assert last_response.ok? + + Exceptionist.esclient.refresh + + exce = UberException.find_all('ExampleProject') + assert_equal 1, exce.count + assert_equal 1, exce.first.occurrences_count + end + + def test_api_minimal_exception + assert_equal [], UberException.find_all('ExampleProject') + + post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/minimal_exception.xml') + assert last_response.ok? + + Exceptionist.esclient.refresh + + exce = UberException.find_all('ExampleProject') + assert_equal 1, exce.count + assert_equal 1, exce.first.occurrences_count end end diff --git a/test/elastic_search_helper.rb b/test/elastic_search_helper.rb new file mode 100644 index 0000000..404ed3a --- /dev/null +++ b/test/elastic_search_helper.rb @@ -0,0 +1,36 @@ +require 'elasticsearch/extensions/test/cluster' +require 'elasticsearch' + +module ElasticSearchHelper + def self.start(port) + Elasticsearch::Extensions::Test::Cluster.start( + cluster_name: "my-testing-cluster", + port: port, + nodes: 1, + ) + + begin + Exceptionist.esclient.delete_indices('exceptionist') + rescue Elasticsearch::Transport::Transport::Errors::NotFound + end + + Exceptionist.esclient.create_indices('exceptionist',{ mappings: { + occurrences: { properties: { + action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + } }, + exceptions:{ properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + } } + } } ) + Exceptionist.esclient.refresh + + end + + def self.stop(port) + Elasticsearch::Extensions::Test::Cluster.stop(port: port) + end +end diff --git a/test/fixtures/exception_with_hash_in_params.xml b/test/fixtures/exception_with_hash_in_params.xml index b9c7081..c756b67 100644 --- a/test/fixtures/exception_with_hash_in_params.xml +++ b/test/fixtures/exception_with_hash_in_params.xml @@ -1,6 +1,6 @@ - ExampleProject + SECRET_API_KEY Hoptoad Notifier 1.2.4 @@ -43,4 +43,4 @@ /testapp production - \ No newline at end of file + diff --git a/test/fixtures/full_exception.xml b/test/fixtures/full_exception.xml index 701d185..4a3763a 100644 --- a/test/fixtures/full_exception.xml +++ b/test/fixtures/full_exception.xml @@ -126,17 +126,17 @@ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10 http - --- -- 883 -- 3 -- 4 -- 2 -- 884 -- 8 -- :gettext_key: Events @top_tabs -:url: /%{domain_id}/events -:id: events - + + - 883 + - 3 + - 4 + - 2 + - 884 + - 8 + - :gettext_key: Events @top_tabs + :url: /events + :id: events + 4f590e004b4107ad63a70da4cab6a971 en-us diff --git a/test/integration_test.rb b/test/integration_test.rb index 05f37fb..4d55ee3 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -7,7 +7,7 @@ config.mode = :rack end -class IntegrationTest < Minitest::Test +class IntegrationTest < AbstractTest include Rack::Test::Methods include Webrat::Methods include Webrat::Matchers @@ -16,10 +16,6 @@ def app ExceptionistApp end - def setup - clear_collections - end - def test_show_a_empty_dashboard visit '/' end @@ -28,6 +24,8 @@ def test_show_the_dashboard_with_one_project occurrence = create_occurrence UberException.occurred(occurrence) + Exceptionist.esclient.refresh + visit '/' assert_contain 'ExampleProject' @@ -35,11 +33,13 @@ def test_show_the_dashboard_with_one_project end def test_show_the_dashboard_with_two_projects - occurrence1 = create_occurrence(:project_name => 'ExampleProject') - UberException.occurred(occurrence1) + occur1 = create_occurrence(project_name: 'ExampleProject') + UberException.occurred(occur1) + + occur2 = create_occurrence(project_name: 'ExampleProject2') + UberException.occurred(occur2) - occurrence2 = create_occurrence(:project_name => 'ExampleProject2') - UberException.occurred(occurrence2) + Exceptionist.esclient.refresh visit '/' assert_contain 'ExampleProject' @@ -50,6 +50,8 @@ def test_with_one_exception UberException.occurred(create_occurrence) UberException.occurred(create_occurrence) + Exceptionist.esclient.refresh + visit '/projects/ExampleProject' assert_contain 'Latest Exceptions for ExampleProject' @@ -59,9 +61,11 @@ def test_with_one_exception def test_with_pagination 27.times do |i| - UberException.occurred(create_occurrence(:action_name => "action_#{i}")) + UberException.occurred(create_occurrence(action_name:"action_#{i}")) end + Exceptionist.esclient.refresh + visit '/projects/ExampleProject' assert_contain 'next page' assert_not_contain 'previous page' @@ -72,8 +76,10 @@ def test_with_pagination end def test_be_sorted_by_most_recent - UberException.occurred(create_occurrence(:action_name => 'show', :occurred_at => '2010-03-01')) - UberException.occurred(create_occurrence(:action_name => 'index', :occurred_at => '2009-02-01')) + UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-03-01')) + UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2009-02-01')) + + Exceptionist.esclient.refresh visit '/projects/ExampleProject' @@ -83,8 +89,10 @@ def test_be_sorted_by_most_recent end def test_show_new_exceptions - UberException.occurred(create_occurrence(:action_name => 'show', :occurred_at => '2010-07-01')) - UberException.occurred(create_occurrence(:action_name => 'index', :occurred_at => '2010-08-01')) + UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-07-01')) + UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2010-08-01')) + + Exceptionist.esclient.refresh visit '/projects/ExampleProject/new_on/2010-07-01?mail_to=the@dude.org' @@ -93,8 +101,10 @@ def test_show_new_exceptions end def test_forget_old_exceptions - UberException.occurred(create_occurrence(:action_name => 'show', :occurred_at => Time.now - (86400 * 50))) - UberException.occurred(create_occurrence(:action_name => 'index', :occurred_at => Time.now)) + UberException.occurred(create_occurrence(action_name:'show', occurred_at:Time.now - (86400 * 50))) + UberException.occurred(create_occurrence(action_name:'index', occurred_at:Time.now)) + + Exceptionist.esclient.refresh visit '/projects/ExampleProject/forget_exceptions', :post @@ -105,6 +115,8 @@ def test_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) + Exceptionist.esclient.refresh + visit "/exceptions/#{occurrence.uber_key}" assert_contain 'GET http://example.com' assert_contain 'NameError: undefined local variable or method dude' @@ -115,14 +127,16 @@ def test_show_a_minimal_occurrence end def test_paginate_occurrences - occurrence1 = create_occurrence(:url => 'http://example.com/?show=one') - occurrence2 = create_occurrence(:url => 'http://example.com/?show=two') - occurrence3 = create_occurrence(:url => 'http://example.com/?show=three') - UberException.occurred(occurrence1) - UberException.occurred(occurrence2) - UberException.occurred(occurrence3) - - visit "/exceptions/#{occurrence1.uber_key}" + occur1 = create_occurrence(url: 'http://example.com/?show=one') + occur2 = create_occurrence(url: 'http://example.com/?show=two') + occur3 = create_occurrence(url: 'http://example.com/?show=three') + UberException.occurred(occur1) + UberException.occurred(occur2) + UberException.occurred(occur3) + + Exceptionist.esclient.refresh + + visit "/exceptions/#{occur1.uber_key}" assert_contain 'Older' assert_contain 'Newer' assert_contain '3 of 3' @@ -141,8 +155,10 @@ def test_paginate_occurrences end def test_be_able_to_close_an_exception - UberException.occurred(create_occurrence(:action_name => 'show')) - UberException.occurred(create_occurrence(:action_name => 'index')) + UberException.occurred(create_occurrence(action_name:'show')) + UberException.occurred(create_occurrence(action_name:'index')) + + Exceptionist.esclient.refresh visit '/projects/ExampleProject' assert_contain 'NameError in users#show' @@ -151,6 +167,9 @@ def test_be_able_to_close_an_exception click_link 'NameError in users#show' click_button 'Close' + + Exceptionist.esclient.refresh + follow_redirect! # redirects back to project page @@ -158,4 +177,5 @@ def test_be_able_to_close_an_exception assert_not_contain 'NameError in users#show' assert_contain 'NameError in users#index' end + end diff --git a/test/mongod-test.conf b/test/mongod-test.conf deleted file mode 100644 index 6c7a270..0000000 --- a/test/mongod-test.conf +++ /dev/null @@ -1,3 +0,0 @@ -dbpath = /tmp/test_mongodb -port = 9736 -bind_ip = 127.0.0.1 diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index bad36e9..8585034 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -1,9 +1,151 @@ require 'test_helper' -class OccurrenceTest < Minitest::Test +class OccurrenceTest < AbstractTest - def setup - clear_collections + def test_delete_all_for + occur = create_occurrence(occurred_at: Time.local(2010, 8, 9)) + create_occurrence(occurred_at: Time.local(2012, 8, 9)) + create_occurrence(occurred_at: Time.local(2011, 8, 9)) + other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal 3, Occurrence.find_all_by_name(occur.project_name).size + + Occurrence.delete_all_for(occur.uber_key) + + Exceptionist.esclient.refresh + + assert_equal 0, Occurrence.find_all_by_name(occur.project_name).size + assert_equal 1, Occurrence.find_all_by_name(other_occur.project_name).size + end + + def test_find_first_for + assert_equal nil, Occurrence.find_first_for('empty db') + + occur = create_occurrence(occurred_at: Time.local(2010, 8, 9)) + create_occurrence(occurred_at: Time.local(2012, 8, 9)) + other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal occur, Occurrence.find_first_for(occur.uber_key) + assert_equal other_occur, Occurrence.find_first_for(other_occur.uber_key) + end + + def test_find_last_for + assert_equal nil, Occurrence.find_last_for('empty db') + + create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal occur, Occurrence.find_last_for(occur.uber_key) + assert_equal other_occur, Occurrence.find_last_for(other_occur.uber_key) + end + + def test_count_all_on + create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) + create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) + create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 10)) + assert_equal 2, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) + + create_occurrence(occurred_at: Time.local(2011, 8, 9, 9, 42)) + + Exceptionist.esclient.refresh + + assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 8)) + assert_equal 3, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) + end + + def test_find_all + occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) + occur4 = create_occurrence(occurred_at: Time.local(2011, 8, 10), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal [occur2, occur4, occur3, occur1], Occurrence.find_all + end + + def test_find_all_by_name + occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) + create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal [occur2, occur3, occur1], Occurrence.find_all_by_name('ExampleProject', 5) + assert_equal [occur2, occur3], Occurrence.find_all_by_name('ExampleProject', 2) + end + + def test_generate_uber_key + base_key = build_occurrence.uber_key + assert_equal base_key, build_occurrence(url: 'lala.com').uber_key + assert_equal base_key, build_occurrence(session: {user_id: 17}).uber_key + + refute_equal base_key, build_occurrence(exception_class: 'NoMethodError').uber_key + end + + def test_generate_uber_key_for_occurrences_in_different_projects + project1_key = build_occurrence(project_name: 'project1').uber_key + project2_key = build_occurrence(project_name: 'project2').uber_key + + refute_equal project1_key, project2_key + end + + def test_generate_uber_key_for_the_same_NoMethodError + key1 = build_occurrence(exception_class: 'NoMethodError', + exception_message: "NoMethodError: undefined method `service' for #").uber_key + key2 = build_occurrence(exception_class: 'NoMethodError', + exception_message: "NoMethodError: undefined method `service' for #").uber_key + + assert_equal key1, key2 + end + + def test_generate_different_uber_keys_for_different_NoMethodErrors + key1 = build_occurrence(exception_class: 'NoMethodError', + exception_message: "NoMethodError: undefined method `service' for #", + exception_backtrace: ["[GEM_ROOT]/gems/activerecord-2.3.4/lib/active_record/attribute_methods.rb:260:in `method_missing'"]).uber_key + key2 = build_occurrence(exception_class: 'NoMethodError', + exception_message: "NoMethodError: undefined method `name' for nil:NilClass", + exception_backtrace: ["[PROJECT_ROOT]/app/models/post.rb:184:in `name'"]).uber_key + + refute_equal key1, key2 + end + + def test_aggregate_RuntimeErrors + runtime_exception = { exception_class: 'RuntimeError' } + + base_key = build_occurrence(runtime_exception).uber_key + assert_equal base_key, build_occurrence(runtime_exception.merge(controller_name: 'projects')).uber_key + assert_equal base_key, build_occurrence(runtime_exception.merge(action_name: 'index')).uber_key + end + + def test_aggregate_TimeoutErrors + timeout_backtrace = [ + "/opt/ruby-enterprise/lib/ruby/1.8/net/protocol.rb:135:in `call'", + "[PROJECT_ROOT]/vendor/plugins/acts_as_solr/lib/solr/connection.rb:158:in `post'" + ] + timeout_exception = { exception_class: 'Timeout::Error', exception_backtrace: timeout_backtrace } + + base_key = build_occurrence(timeout_exception).uber_key + assert_equal base_key, build_occurrence(timeout_exception.merge(controller_name: 'projects')).uber_key + assert_equal base_key, build_occurrence(timeout_exception.merge(action_name: 'index')).uber_key + + timeout_backtrace.insert(0, "/opt/ruby-enterprise/lib/ruby/gems/1.8/gems/system_timer-1.0/lib/system_timer.rb:42:in `install_ruby_sigalrm_handler'") + assert_equal base_key, build_occurrence(timeout_exception.merge(exception_backtrace: timeout_backtrace)).uber_key + + timeout_backtrace[2] = "[PROJECT_ROOT]/app/models/user.rb:158:in `post'" + refute_equal base_key, build_occurrence(timeout_exception.merge(exception_backtrace: timeout_backtrace)).uber_key end def test_parse_a_exception @@ -65,73 +207,4 @@ def test_parse_an_exception_with_hash_in_params assert_equal(hash_in_params, occurrence.parameters) end - def test_an_occurrence_occurred - occurrence = create_occurrence - UberException.occurred(occurrence) - - uber_exp = occurrence.uber_exception - assert_equal 1, uber_exp.occurrences_count - assert_equal 'NameError: undefined local variable or method dude', uber_exp.first_occurrence.exception_message - end - - def test_generate_uber_key - base_key = build_occurrence.uber_key - assert_equal base_key, build_occurrence(:url => 'lala.com').uber_key - assert_equal base_key, build_occurrence(:session => {:user_id => 17}).uber_key - - refute_equal base_key, build_occurrence(:exception_class => 'NoMethodError').uber_key - end - - def test_generate_uber_key_for_occurrences_in_different_projects - project1_key = build_occurrence(:project_name => 'project1').uber_key - project2_key = build_occurrence(:project_name => 'project2').uber_key - - refute_equal project1_key, project2_key - end - - def test_generate_uber_key_for_the_same_NoMethodError - key1 = build_occurrence(:exception_class => 'NoMethodError', - :exception_message => "NoMethodError: undefined method `service' for #").uber_key - key2 = build_occurrence(:exception_class => 'NoMethodError', - :exception_message => "NoMethodError: undefined method `service' for #").uber_key - - assert_equal key1, key2 - end - - def test_generate_different_uber_keys_for_different_NoMethodErrors - key1 = build_occurrence(:exception_class => 'NoMethodError', - :exception_message => "NoMethodError: undefined method `service' for #", - :exception_backtrace => ["[GEM_ROOT]/gems/activerecord-2.3.4/lib/active_record/attribute_methods.rb:260:in `method_missing'"]).uber_key - key2 = build_occurrence(:exception_class => 'NoMethodError', - :exception_message => "NoMethodError: undefined method `name' for nil:NilClass", - :exception_backtrace => ["[PROJECT_ROOT]/app/models/post.rb:184:in `name'"]).uber_key - - refute_equal key1, key2 - end - - def test_aggregate_RuntimeErrors - runtime_exception = { :exception_class => 'RuntimeError' } - - base_key = build_occurrence(runtime_exception).uber_key - assert_equal base_key, build_occurrence(runtime_exception.merge(:controller_name => 'projects')).uber_key - assert_equal base_key, build_occurrence(runtime_exception.merge(:action_name => 'index')).uber_key - end - - def test_aggregate_TimeoutErrors - timeout_backtrace = [ - "/opt/ruby-enterprise/lib/ruby/1.8/net/protocol.rb:135:in `call'", - "[PROJECT_ROOT]/vendor/plugins/acts_as_solr/lib/solr/connection.rb:158:in `post'" - ] - timeout_exception = { :exception_class => 'Timeout::Error', :exception_backtrace => timeout_backtrace } - - base_key = build_occurrence(timeout_exception).uber_key - assert_equal base_key, build_occurrence(timeout_exception.merge(:controller_name => 'projects')).uber_key - assert_equal base_key, build_occurrence(timeout_exception.merge(:action_name => 'index')).uber_key - - timeout_backtrace.insert(0, "/opt/ruby-enterprise/lib/ruby/gems/1.8/gems/system_timer-1.0/lib/system_timer.rb:42:in `install_ruby_sigalrm_handler'") - assert_equal base_key, build_occurrence(timeout_exception.merge(:exception_backtrace => timeout_backtrace)).uber_key - - timeout_backtrace[2] = "[PROJECT_ROOT]/app/models/user.rb:158:in `post'" - refute_equal base_key, build_occurrence(timeout_exception.merge(:exception_backtrace => timeout_backtrace)).uber_key - end end diff --git a/test/project_test.rb b/test/project_test.rb new file mode 100644 index 0000000..df5f91c --- /dev/null +++ b/test/project_test.rb @@ -0,0 +1,32 @@ +require 'test_helper' + +class ProjectTest < AbstractTest + + def test_last_n_days + assert_equal 2, Project.last_n_days(2).size + assert_equal 4, Project.last_n_days(4).size + end + + def test_last_thirty_days + project = Project.new('ExampleProject') + + project.last_thirty_days.each do |day, occur| + assert_equal 0, occur + end + + create_occurrence + + Exceptionist.esclient.refresh + + assert_equal 1, project.last_thirty_days.last[1] + end + + def test_find_by_key + assert_equal nil, Project.find_by_key('test_key') + + Exceptionist.add_project('project', 'test_key') + + assert_equal 'project', Project.find_by_key('test_key').name + end + +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 98940ac..dde71ad 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,33 +1,19 @@ -$LOAD_PATH.unshift File.dirname(File.expand_path(__FILE__)) + '/../lib' - +require 'elastic_search_helper' require 'app' -## -# start our own mongodb when the tests start, -# kill it when they end -# -at_exit do - next if $! +port = 10000 - pid = `ps -e -o pid,command | grep [m]ongod-test`.split(" ")[0] - puts "Killing test mongod server..." - Process.kill("KILL", pid.to_i) - `rm -rf /tmp/test_mongodb` - exit exit_code +at_exit do + ElasticSearchHelper.stop(port) end # minitest install its own at_exit, so we need to do this after our own require 'minitest/autorun' -puts 'Starting mongod for testing at localhost:9736...' - -`mkdir -p /tmp/test_mongodb` -test_dir = File.dirname(File.expand_path(__FILE__)) -`mongod run --fork --logpath /dev/null --config #{test_dir}/mongod-test.conf` -sleep 1 +Exceptionist.endpoint = "localhost:#{port}" +ElasticSearchHelper.start(port) # Configure -Exceptionist.mongo = 'localhost:9736' Exceptionist.add_project 'ExampleProject', 'SECRET_API_KEY' Exceptionist.add_project 'ExampleProject2', 'ANOTHER_SECRET_API_KEY' @@ -40,13 +26,13 @@ def read_fixtures_file(path) def build_occurrence(attributes = {}) default_attributes = { - :exception_class => 'NameError', - :exception_message => 'NameError: undefined local variable or method dude', - :exception_backtrace => ["[PROJECT_ROOT]/app/models/user.rb:53:in `public'", "[PROJECT_ROOT]/app/controllers/users_controller.rb:14:in `show'"], - :controller_name => 'users', - :action_name => 'show', - :project_name => 'ExampleProject', - :url => 'http://example.com' + exception_class: 'NameError', + exception_message: 'NameError: undefined local variable or method dude', + exception_backtrace: ["[PROJECT_ROOT]/app/models/user.rb:53:in `public'", "[PROJECT_ROOT]/app/controllers/users_controller.rb:14:in `show'"], + controller_name: 'users', + action_name: 'show', + project_name: 'ExampleProject', + url: 'http://example.com' } Occurrence.new(default_attributes.merge(attributes)) end @@ -56,6 +42,11 @@ def create_occurrence(attributes = {}) end def clear_collections - Exceptionist.mongo.drop_collection('occurrences') - Exceptionist.mongo.drop_collection('exceptions') + Exceptionist.esclient.delete_by_query( match_all: {}) +end + +class AbstractTest < Minitest::Test + def setup + clear_collections + end end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index bf10cbb..959b842 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -1,55 +1,186 @@ require 'test_helper' -class UberExceptionTest < Minitest::Test - def setup - clear_collections +class UberExceptionTest < AbstractTest + + def test_count_all + UberException.occurred(create_occurrence()) + + Exceptionist.esclient.refresh + + assert_equal 1, UberException.count_all('ExampleProject') + + UberException.occurred(create_occurrence(action_name: 'other')) + UberException.occurred(create_occurrence(project_name: 'OtherProject')) + + Exceptionist.esclient.refresh + + assert_equal 2, UberException.count_all('ExampleProject') + end + + def test_find + uber_exception = UberException.occurred(create_occurrence()) + + Exceptionist.esclient.refresh + + assert_equal uber_exception.id, UberException.find(uber_exception.id).id + end + + def test_find_all + exce1 = UberException.occurred(create_occurrence()) + UberException.occurred(create_occurrence()) + exce2 = UberException.occurred(create_occurrence(action_name: 'other')) + + Exceptionist.esclient.refresh + + assert UberException.find_all('ExampleProject').include? exce1 + assert UberException.find_all('ExampleProject').include? exce2 end - def test_find_all_new_on_a_day + def test_find_all_sorted_by_time + exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) + exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) + exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14, 14, 42), action_name: 'action4')) + + Exceptionist.esclient.refresh + + assert_equal [exce4, exce3, exce2, exce1], UberException.find_all_sorted_by_time('ExampleProject', 0, 10) + assert_equal [exce2], UberException.find_all_sorted_by_time('ExampleProject', 2, 1) + end + + def test_find_all_sorted_by_occurrences_count + exce1 = UberException.occurred(create_occurrence()) + UberException.occurred(create_occurrence()) + exce2 = UberException.occurred(create_occurrence(action_name: 'other')) + + Exceptionist.esclient.refresh + + assert_equal [exce1, exce2], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 0, 10) + assert_equal [exce2], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 1, 10) + end + + def test_find_new_on project = Project.new('ExampleProject') - old_ocr = create_occurrence(:occurred_at => Time.local(2011, 8, 9, 14, 42)) - yesterday_ocr1 = create_occurrence(:action_name => 'index', :occurred_at => Time.local(2011, 8, 12, 14, 42)) - yesterday_ocr2 = create_occurrence(:action_name => 'index', :occurred_at => Time.local(2011, 8, 12, 15, 42)) - today_ocr = create_occurrence(:action_name => 'create', :occurred_at => Time.local(2011, 8, 13, 14, 42)) + old_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) + yesterday_occur1 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 12, 14, 42)) + yesterday_occur2 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 12, 15, 42)) + today_occur1 = create_occurrence(action_name: 'create', occurred_at: Time.local(2011, 8, 13, 14, 42)) + today_occur2 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 13, 14, 42)) + + UberException.occurred(old_occur) + UberException.occurred(yesterday_occur1) + UberException.occurred(yesterday_occur2) + UberException.occurred(today_occur1) + UberException.occurred(today_occur2) - UberException.occurred(old_ocr) - UberException.occurred(yesterday_ocr1) - UberException.occurred(yesterday_ocr2) - UberException.occurred(today_ocr) + Exceptionist.esclient.refresh exceptions = UberException.find_new_on(project.name, Time.local(2011, 8, 12)) assert_equal 1, exceptions.size - assert_equal [yesterday_ocr1.uber_exception], exceptions + assert_equal [yesterday_occur1.uber_exception], exceptions end def test_forget_old_exceptions project = Project.new('ExampleProject') very_old_date = Time.now - (86400 * 50) - very_old_exc = UberException.occurred(create_occurrence(:occurred_at => very_old_date)) - old_exc = UberException.occurred(create_occurrence(:action_name => 'index', :occurred_at => Time.now - (86400 * 28))) - today_exc = UberException.occurred(create_occurrence(:action_name => 'create', :occurred_at => Time.now)) + very_old_exec = UberException.occurred(create_occurrence(occurred_at: very_old_date)) + old_exec = UberException.occurred(create_occurrence(action_name: 'index', occurred_at: Time.now - (86400 * 28))) + today_exec = UberException.occurred(create_occurrence(action_name: 'create', occurred_at: Time.now)) + + Exceptionist.esclient.refresh - assert_equal [today_exc, old_exc, very_old_exc], UberException.find_all_sorted_by_time(project.name, 0, 20) - assert_equal [very_old_exc], UberException.find_new_on(project.name, very_old_date - 60) + assert_equal [today_exec, old_exec, very_old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # shouldn't forget anything UberException.forget_old_exceptions(project.name, 51) - assert_equal [today_exc, old_exc, very_old_exc], UberException.find_all_sorted_by_time(project.name, 0, 20) - assert_equal [very_old_exc], UberException.find_new_on(project.name, very_old_date - 60) + Exceptionist.esclient.refresh + + assert_equal [today_exec, old_exec, very_old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # should forget the very_old exception UberException.forget_old_exceptions(project.name, 30) - assert_equal [today_exc, old_exc], UberException.find_all_sorted_by_time(project.name, 0, 20) + Exceptionist.esclient.refresh + + assert_equal [today_exec, old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) assert_equal [], UberException.find_new_on(project.name, very_old_date - 60) # should forget even more UberException.forget_old_exceptions(project.name, 1) - assert_equal [today_exc], UberException.find_all_sorted_by_time(project.name, 0, 20) + Exceptionist.esclient.refresh + + assert_equal [today_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) assert_equal [], UberException.find_new_on(project.name, Time.now - 86400 - 3600) end + + def test_forget! + uber_exce = UberException.occurred(create_occurrence()) + uber_exce.forget! + + Exceptionist.esclient.refresh + + assert_raises(Elasticsearch::Transport::Transport::Errors::NotFound) do + UberException.find(uber_exce.id) + end + end + + def test_close! + uber_exce = UberException.occurred(create_occurrence()) + uber_exce.close! + + Exceptionist.esclient.refresh + + assert_equal [], UberException.find_all('ExampleProject') + end + + def test_current_occurrence + ocr1 = create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42)) + ocr2 = create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42)) + uber_ex = UberException.occurred(ocr1) + UberException.occurred(ocr2) + + Exceptionist.esclient.refresh + + assert_equal ocr1, uber_ex.current_occurrence(1) + assert_equal ocr2, uber_ex.current_occurrence(2) + + assert_equal nil, uber_ex.current_occurrence(0) + assert_equal nil, uber_ex.current_occurrence(3) + end + + def test_update_occurence_count + uber_ex = UberException.occurred(create_occurrence()) + UberException.occurred(create_occurrence()) + + Exceptionist.esclient.refresh + + Exceptionist.esclient.update('exceptions', uber_ex.id, { doc: { occurrences_count: 42 } }) + + Exceptionist.esclient.refresh + + assert_equal 42, Exceptionist.esclient.get_exception(uber_ex.id).occurrences_count + + uber_ex.update_occurrences_count + + Exceptionist.esclient.refresh + + assert_equal 2, Exceptionist.esclient.get_exception(uber_ex.id).occurrences_count + end + + def test_occurrences_count_on + uber_ex = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42))) + + Exceptionist.esclient.refresh + + assert_equal 2, uber_ex.occurrences_count_on(Time.local(2011, 8, 12)) + assert_equal 1, uber_ex.occurrences_count_on(Time.local(2011, 8, 13)) + end end From 8ff2473518f94e6a5499323da372af82cd6fe540 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 3 Sep 2014 16:28:19 +0200 Subject: [PATCH 007/293] add config file for elasticsearch db --- elasticsearch.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 elasticsearch.yml diff --git a/elasticsearch.yml b/elasticsearch.yml new file mode 100644 index 0000000..32293c4 --- /dev/null +++ b/elasticsearch.yml @@ -0,0 +1,3 @@ +{ + script.disable_dynamic: false +} From d92503c8fee0640e807a790c5b2201e02eac5457 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 3 Sep 2014 16:33:22 +0200 Subject: [PATCH 008/293] add file to create db and insert test data --- test/insert_data.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/insert_data.rb diff --git a/test/insert_data.rb b/test/insert_data.rb new file mode 100644 index 0000000..1c71ba3 --- /dev/null +++ b/test/insert_data.rb @@ -0,0 +1,29 @@ +require './lib/es_client' + +esclient = ESClient.new("localhost:9200") + +# create database +begin + esclient.delete_indices('exceptionist') +rescue Elasticsearch::Transport::Transport::Errors::NotFound +end + +esclient.create_indices('exceptionist', + { mappings: { + occurrences: { properties: { + action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + } }, + exceptions:{ properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + } } + } }) +esclient.refresh + +5.times { `curl -X POST -d @test/fixtures/full_exception.xml http://localhost:9292/notifier_api/v2/notices/?` } + +esclient.refresh + From 8717b248dad678e24d134159e371ca09095bd2bb Mon Sep 17 00:00:00 2001 From: Florian Munz Date: Thu, 4 Sep 2014 15:10:27 +0200 Subject: [PATCH 009/293] bring back tools --- Rakefile | 17 +++++++++++++++++ lib/tools.rb | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 lib/tools.rb diff --git a/Rakefile b/Rakefile index b95533e..e7e8746 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ $LOAD_PATH.unshift 'lib' +require 'tools' task :default => :test @@ -7,3 +8,19 @@ Rake::TestTask.new do |test| test.libs << "test" test.test_files = FileList['test/**/*_test.rb'] end + +desc "Remove a single exception with all occurrences completely" +task :remove_exception do + uber_key = ENV['KEY'] + Exceptionist::Remover.run(uber_key) +end + +desc "Export occurrences in json" +task :export do + Exceptionist::Exporter.run +end + +desc "Import occurrences from json file" +task :import do + Exceptionist::Importer.run +end diff --git a/lib/tools.rb b/lib/tools.rb new file mode 100644 index 0000000..30da55e --- /dev/null +++ b/lib/tools.rb @@ -0,0 +1,39 @@ +require 'boot' +require './config' + +module Exceptionist + class Remover + def self.run(uber_key) + UberException.find(uber_key).forget! + end + end + + class Exporter + def self.run + occurrences = Occurrence.find_all(nil, 10000).map { |occurrence| occurrence.to_hash } + + File.open('occurrences_export.json', 'w') do |file| + file.write(Yajl::Encoder.encode(occurrences)) + end + end + end + + class Importer + def self.run + files = Dir.glob('occurrences_export*').sort + files.each do |file| + puts "importing #{file}" + + occurrences = Yajl::Parser.parse(File.read(file)) + occurrences.each do |occurrence_hash| + occurrence_hash.delete('uber_key') + occurrence_hash.delete('id') + occurrence = Occurrence.new(occurrence_hash) + occurrence.save + + UberException.occurred(occurrence) + end + end + end + end +end From 934b522883a400b32a66fa9f98bd0b67908bae7c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:00:50 +0200 Subject: [PATCH 010/293] add cleardb command to rake --- Rakefile | 5 +++++ lib/tools.rb | 23 +++++++++++++++++++++++ test/insert_data.rb | 29 ----------------------------- 3 files changed, 28 insertions(+), 29 deletions(-) delete mode 100644 test/insert_data.rb diff --git a/Rakefile b/Rakefile index e7e8746..ad7b355 100644 --- a/Rakefile +++ b/Rakefile @@ -24,3 +24,8 @@ desc "Import occurrences from json file" task :import do Exceptionist::Importer.run end + +desc "Clear DB and create index with mapping" +task :cleardb do + Exceptionist::ClearDB.run +end diff --git a/lib/tools.rb b/lib/tools.rb index 30da55e..cd4a87a 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -35,5 +35,28 @@ def self.run end end end + + class ClearDB + def self.run + begin + Exceptionist.esclient.delete_indices('exceptionist') + rescue Elasticsearch::Transport::Transport::Errors::NotFound + end + + Exceptionist.esclient.create_indices('exceptionist', + { mappings: { + occurrences: { properties: { + action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + } }, + exceptions:{ properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + } } + } }) + Exceptionist.esclient.refresh + end end end diff --git a/test/insert_data.rb b/test/insert_data.rb deleted file mode 100644 index 1c71ba3..0000000 --- a/test/insert_data.rb +++ /dev/null @@ -1,29 +0,0 @@ -require './lib/es_client' - -esclient = ESClient.new("localhost:9200") - -# create database -begin - esclient.delete_indices('exceptionist') -rescue Elasticsearch::Transport::Transport::Errors::NotFound -end - -esclient.create_indices('exceptionist', - { mappings: { - occurrences: { properties: { - action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - } }, - exceptions:{ properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - } } - } }) -esclient.refresh - -5.times { `curl -X POST -d @test/fixtures/full_exception.xml http://localhost:9292/notifier_api/v2/notices/?` } - -esclient.refresh - From c689d32bbca75a5079c79bb0b83c6716d02f40e3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:01:51 +0200 Subject: [PATCH 011/293] add default value {} for query parameter --- lib/es_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index d4df8b9..cc47fa1 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -65,7 +65,7 @@ def delete_indices(index) @es.indices.delete(index: index) end - def create_indices(index, query) + def create_indices(index, query={}) @es.indices.create(index: index, body: query) end From d8ec9139602a60ab79215f62fcd2c7806d570ccf Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:02:31 +0200 Subject: [PATCH 012/293] add get_mapping function to esclient --- lib/es_client.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/es_client.rb b/lib/es_client.rb index cc47fa1..4f42395 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -69,6 +69,10 @@ def create_indices(index, query={}) @es.indices.create(index: index, body: query) end + def get_mapping(type) + @es.indices.get_mapping(index: INDEX, type: type) + end + def get_exception(id) response = @es.get(index: INDEX, type: TYPE_EXCEPTIONS, id: id) create_exception(response) From d4e05e3b5bb033c4cfe1ff666a218113ba7f04c1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:04:36 +0200 Subject: [PATCH 013/293] fix method call to findall --- lib/tools.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tools.rb b/lib/tools.rb index cd4a87a..381f033 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -10,7 +10,7 @@ def self.run(uber_key) class Exporter def self.run - occurrences = Occurrence.find_all(nil, 10000).map { |occurrence| occurrence.to_hash } + occurrences = Occurrence.find_all.map { |occurrence| occurrence.to_hash } File.open('occurrences_export.json', 'w') do |file| file.write(Yajl::Encoder.encode(occurrences)) From 07919c311b4afe82314bab64a5a2e11c08b0e5e3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:05:45 +0200 Subject: [PATCH 014/293] move test occurences in fixtures folder --- lib/tools.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tools.rb b/lib/tools.rb index 381f033..7eb6700 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -20,7 +20,7 @@ def self.run class Importer def self.run - files = Dir.glob('occurrences_export*').sort + files = Dir.glob('test/fixtures/occurrences_export*').sort files.each do |file| puts "importing #{file}" From c3061450875a6028345c0e086606290a07b209d7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 13:06:21 +0200 Subject: [PATCH 015/293] modify hash that es can index it as a document --- lib/tools.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/tools.rb b/lib/tools.rb index 7eb6700..3032f1f 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -26,8 +26,14 @@ def self.run occurrences = Yajl::Parser.parse(File.read(file)) occurrences.each do |occurrence_hash| + + # TODO: still problems with es mapping when indexing new documents + replace_empty_deep!(occurrence_hash) occurrence_hash.delete('uber_key') occurrence_hash.delete('id') + occurrence_hash['parameters'].delete('utm_source') if occurrence_hash['parameters'] + occurrence_hash['parameters'].delete('status') if occurrence_hash['parameters'] + occurrence = Occurrence.new(occurrence_hash) occurrence.save @@ -36,6 +42,17 @@ def self.run end end + def self.replace_empty_deep!(h) + h.each do | k, v | + if v && v.empty? + h[k] = nil + else + replace_empty_deep!(v) if v.kind_of?(Hash) + end + end + end + end + class ClearDB def self.run begin From e4c40a54a9ddd99c712a9243232fc3590154060c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 15:37:45 +0200 Subject: [PATCH 016/293] drop datefields with string 'custom-date' --- lib/tools.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/tools.rb b/lib/tools.rb index 3032f1f..288815e 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -44,6 +44,12 @@ def self.run def self.replace_empty_deep!(h) h.each do | k, v | + + # TODO: remove custom date from date fields + if v && v == 'custom-date' + h[k] = nil + end + if v && v.empty? h[k] = nil else From 52c8db88313b43d152377cba4e66dc5aa1102e8e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 15:38:56 +0200 Subject: [PATCH 017/293] break long links in div --- public/stylesheets/exceptionist.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index abdc4f9..0cb6819 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -1,3 +1,7 @@ +html{ + word-wrap: break-word; +} + #project_nav { margin-top: 12px; font-weight: bold; From 4365007b288341d044be16f1b13a3f706bdcd785 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 8 Sep 2014 16:16:57 +0200 Subject: [PATCH 018/293] update flot js to version 0.8.3 --- public/javascripts/jquery-1.4.1.min.js | 152 --------------------- public/javascripts/jquery.flot.min.js | 14 +- public/javascripts/jquery.flot.time.min.js | 7 + public/javascripts/jquery.min.js | 5 + views/layout.erb | 3 +- 5 files changed, 22 insertions(+), 159 deletions(-) delete mode 100644 public/javascripts/jquery-1.4.1.min.js create mode 100644 public/javascripts/jquery.flot.time.min.js create mode 100644 public/javascripts/jquery.min.js diff --git a/public/javascripts/jquery-1.4.1.min.js b/public/javascripts/jquery-1.4.1.min.js deleted file mode 100644 index 0c7294c..0000000 --- a/public/javascripts/jquery-1.4.1.min.js +++ /dev/null @@ -1,152 +0,0 @@ -/*! - * jQuery JavaScript Library v1.4.1 - * http://jquery.com/ - * - * Copyright 2010, John Resig - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * Includes Sizzle.js - * http://sizzlejs.com/ - * Copyright 2010, The Dojo Foundation - * Released under the MIT, BSD, and GPL Licenses. - * - * Date: Mon Jan 25 19:43:33 2010 -0500 - */ -(function(z,v){function la(){if(!c.isReady){try{r.documentElement.doScroll("left")}catch(a){setTimeout(la,1);return}c.ready()}}function Ma(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function X(a,b,d,f,e,i){var j=a.length;if(typeof b==="object"){for(var n in b)X(a,n,b[n],f,e,d);return a}if(d!==v){f=!i&&f&&c.isFunction(d);for(n=0;n-1){i=j.data;i.beforeFilter&&i.beforeFilter[a.type]&&!i.beforeFilter[a.type](a)||f.push(j.selector)}else delete x[o]}i=c(a.target).closest(f, -a.currentTarget);m=0;for(s=i.length;m)[^>]*$|^#([\w-]+)$/,Qa=/^.[^:#\[\.,]*$/,Ra=/\S/,Sa=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Ta=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,O=navigator.userAgent, -va=false,P=[],L,$=Object.prototype.toString,aa=Object.prototype.hasOwnProperty,ba=Array.prototype.push,Q=Array.prototype.slice,wa=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(typeof a==="string")if((d=Pa.exec(a))&&(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:r;if(a=Ta.exec(a))if(c.isPlainObject(b)){a=[r.createElement(a[1])];c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=ra([d[1]], -[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}}else{if(b=r.getElementById(d[2])){if(b.id!==d[2])return S.find(a);this.length=1;this[0]=b}this.context=r;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=r;a=r.getElementsByTagName(a)}else return!b||b.jquery?(b||S).find(a):c(b).find(a);else if(c.isFunction(a))return S.ready(a);if(a.selector!==v){this.selector=a.selector;this.context=a.context}return c.isArray(a)?this.setArray(a):c.makeArray(a, -this)},selector:"",jquery:"1.4.1",length:0,size:function(){return this.length},toArray:function(){return Q.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){a=c(a||null);a.prevObject=this;a.context=this.context;if(b==="find")a.selector=this.selector+(this.selector?" ":"")+d;else if(b)a.selector=this.selector+"."+b+"("+d+")";return a},setArray:function(a){this.length=0;ba.apply(this,a);return this},each:function(a,b){return c.each(this, -a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(r,c);else P&&P.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(Q.apply(this,arguments),"slice",Q.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this,function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject||c(null)},push:ba,sort:[].sort,splice:[].splice}; -c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,i,j,n;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
a";var e=d.getElementsByTagName("*"),i=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!i)){c.support= -{leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(i.getAttribute("style")),hrefNormalized:i.getAttribute("href")==="/a",opacity:/^0.55$/.test(i.style.opacity),cssFloat:!!i.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:r.createElement("select").appendChild(r.createElement("option")).selected,checkClone:false,scriptEval:false,noCloneEvent:true,boxModel:null}; -b.type="text/javascript";try{b.appendChild(r.createTextNode("window."+f+"=1;"))}catch(j){}a.insertBefore(b,a.firstChild);if(z[f]){c.support.scriptEval=true;delete z[f]}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function n(){c.support.noCloneEvent=false;d.detachEvent("onclick",n)});d.cloneNode(true).fireEvent("onclick")}d=r.createElement("div");d.innerHTML="";a=r.createDocumentFragment();a.appendChild(d.firstChild); -c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var n=r.createElement("div");n.style.width=n.style.paddingLeft="1px";r.body.appendChild(n);c.boxModel=c.support.boxModel=n.offsetWidth===2;r.body.removeChild(n).style.display="none"});a=function(n){var o=r.createElement("div");n="on"+n;var m=n in o;if(!m){o.setAttribute(n,"return;");m=typeof o[n]==="function"}return m};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=i=null}})();c.props= -{"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var G="jQuery"+J(),Ua=0,xa={},Va={};c.extend({cache:{},expando:G,noData:{embed:true,object:true,applet:true},data:function(a,b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var f=a[G],e=c.cache;if(!b&&!f)return null;f||(f=++Ua);if(typeof b==="object"){a[G]=f;e=e[f]=c.extend(true, -{},b)}else e=e[f]?e[f]:typeof d==="undefined"?Va:(e[f]={});if(d!==v){a[G]=f;e[b]=d}return typeof b==="string"?e[b]:e}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var d=a[G],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{try{delete a[G]}catch(i){a.removeAttribute&&a.removeAttribute(G)}delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this, -a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===v){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===v&&this.length)f=c.data(this[0],a);return f===v&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this,a,b)})},removeData:function(a){return this.each(function(){c.removeData(this,a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d); -return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===v)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this,a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]|| -a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var ya=/[\n\t]/g,ca=/\s+/,Wa=/\r/g,Xa=/href|src|style/,Ya=/(button|input)/i,Za=/(button|input|object|select|textarea)/i,$a=/^(a|area)$/i,za=/radio|checkbox/;c.fn.extend({attr:function(a,b){return X(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(o){var m= -c(this);m.addClass(a.call(this,o,m.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(ca),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===v){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value|| -{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var i=b?d:0;for(d=b?d+1:e.length;i=0;else if(c.nodeName(this,"select")){var x=c.makeArray(s);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),x)>=0});if(!x.length)this.selectedIndex=-1}else this.value=s}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return v;if(f&&b in c.attrFn)return c(a)[b](d); -f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==v;b=f&&c.props[b]||b;if(a.nodeType===1){var i=Xa.test(b);if(b in a&&f&&!i){if(e){b==="type"&&Ya.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed");a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Za.test(a.nodeName)||$a.test(a.nodeName)&&a.href?0:v;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText= -""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&i?a.getAttribute(b,2):a.getAttribute(b);return a===null?v:a}return c.style(a,b,d)}});var ab=function(a){return a.replace(/[^\w\s\.\|`]/g,function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType===3||a.nodeType===8)){if(a.setInterval&&a!==z&&!a.frameElement)a=z;if(!d.guid)d.guid=c.guid++;if(f!==v){d=c.proxy(d);d.data=f}var e=c.data(a,"events")||c.data(a,"events",{}),i=c.data(a,"handle"),j;if(!i){j= -function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(j.elem,arguments):v};i=c.data(a,"handle",j)}if(i){i.elem=a;b=b.split(/\s+/);for(var n,o=0;n=b[o++];){var m=n.split(".");n=m.shift();if(o>1){d=c.proxy(d);if(f!==v)d.data=f}d.type=m.slice(0).sort().join(".");var s=e[n],x=this.special[n]||{};if(!s){s=e[n]={};if(!x.setup||x.setup.call(a,f,m,d)===false)if(a.addEventListener)a.addEventListener(n,i,false);else a.attachEvent&&a.attachEvent("on"+n,i)}if(x.add)if((m=x.add.call(a, -d,f,m,s))&&c.isFunction(m)){m.guid=m.guid||d.guid;m.data=m.data||d.data;m.type=m.type||d.type;d=m}s[d.guid]=d;this.global[n]=true}a=null}}},global:{},remove:function(a,b,d){if(!(a.nodeType===3||a.nodeType===8)){var f=c.data(a,"events"),e,i,j;if(f){if(b===v||typeof b==="string"&&b.charAt(0)===".")for(i in f)this.remove(a,i+(b||""));else{if(b.type){d=b.handler;b=b.type}b=b.split(/\s+/);for(var n=0;i=b[n++];){var o=i.split(".");i=o.shift();var m=!o.length,s=c.map(o.slice(0).sort(),ab);s=new RegExp("(^|\\.)"+ -s.join("\\.(?:.*\\.)?")+"(\\.|$)");var x=this.special[i]||{};if(f[i]){if(d){j=f[i][d.guid];delete f[i][d.guid]}else for(var A in f[i])if(m||s.test(f[i][A].type))delete f[i][A];x.remove&&x.remove.call(a,o,j);for(e in f[i])break;if(!e){if(!x.teardown||x.teardown.call(a,o)===false)if(a.removeEventListener)a.removeEventListener(i,c.data(a,"handle"),false);else a.detachEvent&&a.detachEvent("on"+i,c.data(a,"handle"));e=null;delete f[i]}}}}for(e in f)break;if(!e){if(A=c.data(a,"handle"))A.elem=null;c.removeData(a, -"events");c.removeData(a,"handle")}}}},trigger:function(a,b,d,f){var e=a.type||a;if(!f){a=typeof a==="object"?a[G]?a:c.extend(c.Event(e),a):c.Event(e);if(e.indexOf("!")>=0){a.type=e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();this.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType===8)return v;a.result=v;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(f=c.data(d,"handle"))&&f.apply(d, -b);f=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+e]&&d["on"+e].apply(d,b)===false)a.result=false}catch(i){}if(!a.isPropagationStopped()&&f)c.event.trigger(a,b,f,true);else if(!a.isDefaultPrevented()){d=a.target;var j;if(!(c.nodeName(d,"a")&&e==="click")&&!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()])){try{if(d[e]){if(j=d["on"+e])d["on"+e]=null;this.triggered=true;d[e]()}}catch(n){}if(j)d["on"+e]=j;this.triggered=false}}},handle:function(a){var b, -d;a=arguments[0]=c.event.fix(a||z.event);a.currentTarget=this;d=a.type.split(".");a.type=d.shift();b=!d.length&&!a.exclusive;var f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)");d=(c.data(this,"events")||{})[a.type];for(var e in d){var i=d[e];if(b||f.test(i.type)){a.handler=i;a.data=i.data;i=i.apply(this,arguments);if(i!==v){a.result=i;if(i===false){a.preventDefault();a.stopPropagation()}}if(a.isImmediatePropagationStopped())break}}return a.result},props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "), -fix:function(a){if(a[G])return a;var b=a;a=c.Event(b);for(var d=this.props.length,f;d;){f=this.props[--d];a[f]=b[f]}if(!a.target)a.target=a.srcElement||r;if(a.target.nodeType===3)a.target=a.target.parentNode;if(!a.relatedTarget&&a.fromElement)a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement;if(a.pageX==null&&a.clientX!=null){b=r.documentElement;d=r.body;a.pageX=a.clientX+(b&&b.scrollLeft||d&&d.scrollLeft||0)-(b&&b.clientLeft||d&&d.clientLeft||0);a.pageY=a.clientY+(b&&b.scrollTop|| -d&&d.scrollTop||0)-(b&&b.clientTop||d&&d.clientTop||0)}if(!a.which&&(a.charCode||a.charCode===0?a.charCode:a.keyCode))a.which=a.charCode||a.keyCode;if(!a.metaKey&&a.ctrlKey)a.metaKey=a.ctrlKey;if(!a.which&&a.button!==v)a.which=a.button&1?1:a.button&2?3:a.button&4?2:0;return a},guid:1E8,proxy:c.proxy,special:{ready:{setup:c.bindReady,teardown:c.noop},live:{add:function(a,b){c.extend(a,b||{});a.guid+=b.selector+b.live;b.liveProxy=a;c.event.add(this,b.live,na,b)},remove:function(a){if(a.length){var b= -0,d=new RegExp("(^|\\.)"+a[0]+"(\\.|$)");c.each(c.data(this,"events").live||{},function(){d.test(this.type)&&b++});b<1&&c.event.remove(this,a[0],na)}},special:{}},beforeunload:{setup:function(a,b,d){if(this.setInterval)this.onbeforeunload=d;return false},teardown:function(a,b){if(this.onbeforeunload===b)this.onbeforeunload=null}}}};c.Event=function(a){if(!this.preventDefault)return new c.Event(a);if(a&&a.type){this.originalEvent=a;this.type=a.type}else this.type=a;this.timeStamp=J();this[G]=true}; -c.Event.prototype={preventDefault:function(){this.isDefaultPrevented=Z;var a=this.originalEvent;if(a){a.preventDefault&&a.preventDefault();a.returnValue=false}},stopPropagation:function(){this.isPropagationStopped=Z;var a=this.originalEvent;if(a){a.stopPropagation&&a.stopPropagation();a.cancelBubble=true}},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=Z;this.stopPropagation()},isDefaultPrevented:Y,isPropagationStopped:Y,isImmediatePropagationStopped:Y};var Aa=function(a){for(var b= -a.relatedTarget;b&&b!==this;)try{b=b.parentNode}catch(d){break}if(b!==this){a.type=a.data;c.event.handle.apply(this,arguments)}},Ba=function(a){a.type=a.data;c.event.handle.apply(this,arguments)};c.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(a,b){c.event.special[a]={setup:function(d){c.event.add(this,b,d&&d.selector?Ba:Aa,a)},teardown:function(d){c.event.remove(this,b,d&&d.selector?Ba:Aa)}}});if(!c.support.submitBubbles)c.event.special.submit={setup:function(a,b,d){if(this.nodeName.toLowerCase()!== -"form"){c.event.add(this,"click.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="submit"||i==="image")&&c(e).closest("form").length)return ma("submit",this,arguments)});c.event.add(this,"keypress.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="text"||i==="password")&&c(e).closest("form").length&&f.keyCode===13)return ma("submit",this,arguments)})}else return false},remove:function(a,b){c.event.remove(this,"click.specialSubmit"+(b?"."+b.guid:""));c.event.remove(this, -"keypress.specialSubmit"+(b?"."+b.guid:""))}};if(!c.support.changeBubbles){var da=/textarea|input|select/i;function Ca(a){var b=a.type,d=a.value;if(b==="radio"||b==="checkbox")d=a.checked;else if(b==="select-multiple")d=a.selectedIndex>-1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d}function ea(a,b){var d=a.target,f,e;if(!(!da.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Ca(d);if(a.type!=="focusout"|| -d.type!=="radio")c.data(d,"_change_data",e);if(!(f===v||e===f))if(f!=null||e){a.type="change";return c.event.trigger(a,b,d)}}}c.event.special.change={filters:{focusout:ea,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return ea.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return ea.call(this,a)},beforeactivate:function(a){a= -a.target;a.nodeName.toLowerCase()==="input"&&a.type==="radio"&&c.data(a,"_change_data",Ca(a))}},setup:function(a,b,d){for(var f in T)c.event.add(this,f+".specialChange."+d.guid,T[f]);return da.test(this.nodeName)},remove:function(a,b){for(var d in T)c.event.remove(this,d+".specialChange"+(b?"."+b.guid:""),T[d]);return da.test(this.nodeName)}};var T=c.event.special.change.filters}r.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this, -f)}c.event.special[b]={setup:function(){this.addEventListener(a,d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,f,e){if(typeof d==="object"){for(var i in d)this[b](i,f,d[i],e);return this}if(c.isFunction(f)){e=f;f=v}var j=b==="one"?c.proxy(e,function(n){c(this).unbind(n,j);return e.apply(this,arguments)}):e;return d==="unload"&&b!=="one"?this.one(d,f,e):this.each(function(){c.event.add(this,d,j,f)})}});c.fn.extend({unbind:function(a, -b){if(typeof a==="object"&&!a.preventDefault){for(var d in a)this.unbind(d,a[d]);return this}return this.each(function(){c.event.remove(this,a,b)})},trigger:function(a,b){return this.each(function(){c.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0]){a=c.Event(a);a.preventDefault();a.stopPropagation();c.event.trigger(a,b,this[0]);return a.result}},toggle:function(a){for(var b=arguments,d=1;d0){y=t;break}}t=t[g]}l[q]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,e=0,i=Object.prototype.toString,j=false,n=true;[0,0].sort(function(){n=false;return 0});var o=function(g,h,k,l){k=k||[];var q=h=h||r;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g|| -typeof g!=="string")return k;for(var p=[],u,t,y,R,H=true,M=w(h),I=g;(f.exec(""),u=f.exec(I))!==null;){I=u[3];p.push(u[1]);if(u[2]){R=u[3];break}}if(p.length>1&&s.exec(g))if(p.length===2&&m.relative[p[0]])t=fa(p[0]+p[1],h);else for(t=m.relative[p[0]]?[h]:o(p.shift(),h);p.length;){g=p.shift();if(m.relative[g])g+=p.shift();t=fa(g,t)}else{if(!l&&p.length>1&&h.nodeType===9&&!M&&m.match.ID.test(p[0])&&!m.match.ID.test(p[p.length-1])){u=o.find(p.shift(),h,M);h=u.expr?o.filter(u.expr,u.set)[0]:u.set[0]}if(h){u= -l?{expr:p.pop(),set:A(l)}:o.find(p.pop(),p.length===1&&(p[0]==="~"||p[0]==="+")&&h.parentNode?h.parentNode:h,M);t=u.expr?o.filter(u.expr,u.set):u.set;if(p.length>0)y=A(t);else H=false;for(;p.length;){var D=p.pop();u=D;if(m.relative[D])u=p.pop();else D="";if(u==null)u=h;m.relative[D](y,u,M)}}else y=[]}y||(y=t);y||o.error(D||g);if(i.call(y)==="[object Array]")if(H)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&E(h,y[g])))k.push(t[g])}else for(g=0;y[g]!=null;g++)y[g]&& -y[g].nodeType===1&&k.push(t[g]);else k.push.apply(k,y);else A(y,k);if(R){o(R,q,k,l);o.uniqueSort(k)}return k};o.uniqueSort=function(g){if(C){j=n;g.sort(C);if(j)for(var h=1;h":function(g,h){var k=typeof h==="string";if(k&&!/\W/.test(h)){h=h.toLowerCase();for(var l=0,q=g.length;l=0))k||l.push(u);else if(k)h[p]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()},CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&& -"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,k,l,q,p){h=g[1].replace(/\\/g,"");if(!p&&m.attrMap[h])g[1]=m.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,k,l,q){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=o(g[3],null,null,h);else{g=o.filter(g[3],h,k,true^q);k||l.push.apply(l,g);return false}else if(m.match.POS.test(g[0])||m.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true); -return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,k){return!!o(k[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)},text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"=== -g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}},setFilters:{first:function(g,h){return h===0},last:function(g,h,k,l){return h===l.length-1},even:function(g,h){return h%2=== -0},odd:function(g,h){return h%2===1},lt:function(g,h,k){return hk[3]-0},nth:function(g,h,k){return k[3]-0===h},eq:function(g,h,k){return k[3]-0===h}},filter:{PSEUDO:function(g,h,k,l){var q=h[1],p=m.filters[q];if(p)return p(g,k,h,l);else if(q==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(q==="not"){h=h[3];k=0;for(l=h.length;k= -0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var k=h[1];g=m.attrHandle[k]?m.attrHandle[k](g):g[k]!=null?g[k]:g.getAttribute(k);k=g+"";var l=h[2];h=h[4];return g==null?l==="!=":l==="="?k===h:l==="*="?k.indexOf(h)>=0:l==="~="?(" "+k+" ").indexOf(h)>=0:!h?k&&g!==false:l==="!="?k!==h:l==="^="? -k.indexOf(h)===0:l==="$="?k.substr(k.length-h.length)===h:l==="|="?k===h||k.substr(0,h.length+1)===h+"-":false},POS:function(g,h,k,l){var q=m.setFilters[h[2]];if(q)return q(g,k,h,l)}}},s=m.match.POS;for(var x in m.match){m.match[x]=new RegExp(m.match[x].source+/(?![^\[]*\])(?![^\(]*\))/.source);m.leftMatch[x]=new RegExp(/(^(?:.|\r|\n)*?)/.source+m.match[x].source.replace(/\\(\d+)/g,function(g,h){return"\\"+(h-0+1)}))}var A=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g}; -try{Array.prototype.slice.call(r.documentElement.childNodes,0)}catch(B){A=function(g,h){h=h||[];if(i.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var k=0,l=g.length;k";var k=r.documentElement;k.insertBefore(g,k.firstChild);if(r.getElementById(h)){m.find.ID=function(l,q,p){if(typeof q.getElementById!=="undefined"&&!p)return(q=q.getElementById(l[1]))?q.id===l[1]||typeof q.getAttributeNode!=="undefined"&&q.getAttributeNode("id").nodeValue===l[1]?[q]:v:[]};m.filter.ID=function(l,q){var p=typeof l.getAttributeNode!=="undefined"&&l.getAttributeNode("id"); -return l.nodeType===1&&p&&p.nodeValue===q}}k.removeChild(g);k=g=null})();(function(){var g=r.createElement("div");g.appendChild(r.createComment(""));if(g.getElementsByTagName("*").length>0)m.find.TAG=function(h,k){k=k.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var l=0;k[l];l++)k[l].nodeType===1&&h.push(k[l]);k=h}return k};g.innerHTML="";if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")m.attrHandle.href=function(h){return h.getAttribute("href", -2)};g=null})();r.querySelectorAll&&function(){var g=o,h=r.createElement("div");h.innerHTML="

";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){o=function(l,q,p,u){q=q||r;if(!u&&q.nodeType===9&&!w(q))try{return A(q.querySelectorAll(l),p)}catch(t){}return g(l,q,p,u)};for(var k in g)o[k]=g[k];h=null}}();(function(){var g=r.createElement("div");g.innerHTML="
";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length=== -0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){m.order.splice(1,0,"CLASS");m.find.CLASS=function(h,k,l){if(typeof k.getElementsByClassName!=="undefined"&&!l)return k.getElementsByClassName(h[1])};g=null}}})();var E=r.compareDocumentPosition?function(g,h){return g.compareDocumentPosition(h)&16}:function(g,h){return g!==h&&(g.contains?g.contains(h):true)},w=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},fa=function(g,h){var k=[], -l="",q;for(h=h.nodeType?[h]:h;q=m.match.PSEUDO.exec(g);){l+=q[0];g=g.replace(m.match.PSEUDO,"")}g=m.relative[g]?g+"*":g;q=0;for(var p=h.length;q=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var i=d;i0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,i={},j;if(f&&a.length){e=0;for(var n=a.length;e --1:c(f).is(e)){d.push({selector:j,elem:f});delete i[j]}}f=f.parentNode}}return d}var o=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(m,s){for(;s&&s.ownerDocument&&s!==b;){if(o?o.index(s)>-1:c(s).is(a))return s;s=s.parentNode}return null})},index:function(a){if(!a||typeof a==="string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(), -a);return this.pushStack(pa(a[0])||pa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode",d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")}, -nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);bb.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e): -e;if((this.length>1||db.test(f))&&cb.test(a))e=e.reverse();return this.pushStack(e,a,Q.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===v||a.nodeType!==1||!c(a).is(d));){a.nodeType===1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!== -b&&d.push(a);return d}});var Fa=/ jQuery\d+="(?:\d+|null)"/g,V=/^\s+/,Ga=/(<([\w:]+)[^>]*?)\/>/g,eb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,Ha=/<([\w:]+)/,fb=/"},F={option:[1,""],legend:[1,"
","
"],thead:[1,"","
"],tr:[2,"","
"],td:[3,"","
"], -col:[2,"","
"],area:[1,"",""],_default:[0,"",""]};F.optgroup=F.option;F.tbody=F.tfoot=F.colgroup=F.caption=F.thead;F.th=F.td;if(!c.support.htmlSerialize)F._default=[1,"div
","
"];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==v)return this.empty().append((this[0]&&this[0].ownerDocument||r).createTextNode(a));return c.getText(this)}, -wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this},wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length? -d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments, -false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&& -!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Fa,"").replace(V,"")],f)[0]}else return this.cloneNode(true)});if(a===true){qa(this,b);qa(this.find("*"),b.find("*"))}return b},html:function(a){if(a===v)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(Fa,""):null;else if(typeof a==="string"&&!/ + + From f2b873aaddbdae04214d4b47fe9d370b1a20f504 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 08:51:28 +0200 Subject: [PATCH 019/293] Add message when there are no exceptions --- test/integration_test.rb | 7 ++++++ views/index.erb | 49 ++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 4d55ee3..d46ace1 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -46,6 +46,13 @@ def test_show_the_dashboard_with_two_projects assert_contain 'ExampleProject2' end + def test_with_no_exceptions + visit '/projects/ExampleProject' + + assert_contain 'Latest Exceptions for ExampleProject' + assert_contain 'Are you kidding? There are no exceptions!' + end + def test_with_one_exception UberException.occurred(create_occurrence) UberException.occurred(create_occurrence) diff --git a/views/index.erb b/views/index.erb index 942f027..dcd8e24 100644 --- a/views/index.erb +++ b/views/index.erb @@ -1,30 +1,35 @@

<%= @title %>

-
- River -
+<% if ! @uber_exceptions.empty? %> -
- <%= '↓ ' if params[:sort_by].nil? || params[:sort_by] == 'latest' %>most recent · - <%= '↓ ' if params[:sort_by] == 'frequent' %>most frequent -
+
+ River +
+
+ <%= '↓ ' if params[:sort_by].nil? || params[:sort_by] == 'latest' %>most recent · + <%= '↓ ' if params[:sort_by] == 'frequent' %>most frequent +
-
-
- <%= partial(:exceptions, :exceptions => @uber_exceptions) %> -
+
+
+ <%= partial(:exceptions, :exceptions => @uber_exceptions) %> +
-
- <% if @start - 25 >= 0 %> - « previous page - <% end %>  -
-<% p %> -
- <% if @start + 25 <= @current_project.exceptions_count %> - next page » - <% end %> -
+
+ <% if @start - 25 >= 0 %> + « previous page + <% end %>  +
+ <% p %> +
+ <% if @start + 25 <= @current_project.exceptions_count %> + next page » + <% end %> +
+ +<% else %> +

Are you kidding? There are no exceptions!

+<% end %>
From 58101423ef98615e1404dc990859145eb44da3f1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 12:49:47 +0200 Subject: [PATCH 020/293] create central helper for ES the central helper es_helper is used by tests and rake --- Rakefile | 12 +++- lib/es_helper.rb | 111 ++++++++++++++++++++++++++++++++++ test/elastic_search_helper.rb | 36 ----------- test/test_helper.rb | 7 ++- 4 files changed, 124 insertions(+), 42 deletions(-) create mode 100644 lib/es_helper.rb delete mode 100644 test/elastic_search_helper.rb diff --git a/Rakefile b/Rakefile index ad7b355..695e1c2 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ $LOAD_PATH.unshift 'lib' require 'tools' +require 'es_helper' task :default => :test @@ -17,15 +18,20 @@ end desc "Export occurrences in json" task :export do - Exceptionist::Exporter.run + ESHelper::Exporter.run end desc "Import occurrences from json file" task :import do - Exceptionist::Importer.run + ESHelper::Importer.run end desc "Clear DB and create index with mapping" task :cleardb do - Exceptionist::ClearDB.run + ESHelper::ClearDB.run +end + +desc "Print mapping" +task :mapping do + ESHelper::Mapping.run end diff --git a/lib/es_helper.rb b/lib/es_helper.rb new file mode 100644 index 0000000..e8a71ed --- /dev/null +++ b/lib/es_helper.rb @@ -0,0 +1,111 @@ +require 'elasticsearch/extensions/test/cluster' +require 'elasticsearch' + +module ESHelper + + def self.startCluster(port) + Elasticsearch::Extensions::Test::Cluster.start( + cluster_name: "my-testing-cluster", + port: port, + nodes: 1, + ) + end + + def self.stopCluster(port) + Elasticsearch::Extensions::Test::Cluster.stop(port: port) + end + + class Exporter + def self.run + occurrences = Occurrence.find_all.map { |occurrence| occurrence.to_hash } + + File.open('occurrences_export.json', 'w') do |file| + file.write(Yajl::Encoder.encode(occurrences)) + end + end + end + + class Importer + def self.run + files = Dir.glob('test/fixtures/occurrences_export*').sort + files.each do |file| + puts "importing #{file}" + + occurrences = Yajl::Parser.parse(File.read(file)) + occurrences.each do |occurrence_hash| + + # TODO: still problems with es mapping when indexing new documents + pp occurrence_hash + replace_empty_deep!(occurrence_hash) + occurrence_hash.delete('uber_key') + occurrence_hash.delete('id') + occurrence_hash['parameters'].delete('utm_source') if occurrence_hash['parameters'] + occurrence_hash['parameters'].delete('status') if occurrence_hash['parameters'] + + occurrence = Occurrence.new(occurrence_hash) + occurrence.save + + UberException.occurred(occurrence) + end + end + end + + def self.replace_empty_deep!(h) + h.each do | k, v | + + # TODO: remove custom date from date fields + if v && v == 'custom-date' + h[k] = nil + end + + if v && v.empty? + h[k] = nil + else + replace_empty_deep!(v) if v.kind_of?(Hash) + end + end + end + end + + class ClearDB + def self.run + begin + Exceptionist.esclient.delete_indices('exceptionist') + rescue Elasticsearch::Transport::Transport::Errors::NotFound + end + + Exceptionist.esclient.create_indices('exceptionist', + { mappings: { + _default_: { + dynamic: 'false' + }, + occurrences: { + properties: { + action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + occurred_at_day: { type: 'date' }, + occurred_at: { type: 'date' } + } }, + exceptions:{ + properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + closed: { type: 'boolean' }, + occurred_at: { type: 'date' } + } } + } }) + Exceptionist.esclient.refresh + end + end + + + class Mapping + def self.run + pp Exceptionist.esclient.get_mapping('occurrences') + puts + pp Exceptionist.esclient.get_mapping('exceptions') + end + end +end diff --git a/test/elastic_search_helper.rb b/test/elastic_search_helper.rb deleted file mode 100644 index 404ed3a..0000000 --- a/test/elastic_search_helper.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'elasticsearch/extensions/test/cluster' -require 'elasticsearch' - -module ElasticSearchHelper - def self.start(port) - Elasticsearch::Extensions::Test::Cluster.start( - cluster_name: "my-testing-cluster", - port: port, - nodes: 1, - ) - - begin - Exceptionist.esclient.delete_indices('exceptionist') - rescue Elasticsearch::Transport::Transport::Errors::NotFound - end - - Exceptionist.esclient.create_indices('exceptionist',{ mappings: { - occurrences: { properties: { - action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - } }, - exceptions:{ properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - } } - } } ) - Exceptionist.esclient.refresh - - end - - def self.stop(port) - Elasticsearch::Extensions::Test::Cluster.stop(port: port) - end -end diff --git a/test/test_helper.rb b/test/test_helper.rb index dde71ad..3fd45ff 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,17 +1,18 @@ -require 'elastic_search_helper' +require 'es_helper' require 'app' port = 10000 at_exit do - ElasticSearchHelper.stop(port) + ESHelper.stopCluster end # minitest install its own at_exit, so we need to do this after our own require 'minitest/autorun' Exceptionist.endpoint = "localhost:#{port}" -ElasticSearchHelper.start(port) +ESHelper.startCluster +ESHelper::ClearDB.run # Configure Exceptionist.add_project 'ExampleProject', 'SECRET_API_KEY' From 6b51451b242ad3ec13c6d01b5604ebc54206ed84 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 12:52:48 +0200 Subject: [PATCH 021/293] access host and port over exceptionist --- lib/es_helper.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index e8a71ed..54d8d0a 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -3,16 +3,16 @@ module ESHelper - def self.startCluster(port) + def self.startCluster Elasticsearch::Extensions::Test::Cluster.start( cluster_name: "my-testing-cluster", - port: port, + port: Exceptionist.esclient.port, nodes: 1, ) end - def self.stopCluster(port) - Elasticsearch::Extensions::Test::Cluster.stop(port: port) + def self.stopCluster + Elasticsearch::Extensions::Test::Cluster.stop( port: Exceptionist.esclient.port ) end class Exporter From 40d4691a663aae5fd853d722540f8fd707bde36b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 12:53:24 +0200 Subject: [PATCH 022/293] no need for modification of hash, not indexed the dynamic mapping is disabled so we don't need to modify the hash anymore --- lib/es_helper.rb | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 54d8d0a..69fc052 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -34,13 +34,9 @@ def self.run occurrences = Yajl::Parser.parse(File.read(file)) occurrences.each do |occurrence_hash| - # TODO: still problems with es mapping when indexing new documents pp occurrence_hash - replace_empty_deep!(occurrence_hash) occurrence_hash.delete('uber_key') occurrence_hash.delete('id') - occurrence_hash['parameters'].delete('utm_source') if occurrence_hash['parameters'] - occurrence_hash['parameters'].delete('status') if occurrence_hash['parameters'] occurrence = Occurrence.new(occurrence_hash) occurrence.save @@ -49,22 +45,6 @@ def self.run end end end - - def self.replace_empty_deep!(h) - h.each do | k, v | - - # TODO: remove custom date from date fields - if v && v == 'custom-date' - h[k] = nil - end - - if v && v.empty? - h[k] = nil - else - replace_empty_deep!(v) if v.kind_of?(Hash) - end - end - end end class ClearDB From 81809c32350ec42b92d0d9560c4eb467a57dd9c2 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 12:55:47 +0200 Subject: [PATCH 023/293] remove attr endpoint, can be accessed over esclient --- lib/exceptionist.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/exceptionist.rb b/lib/exceptionist.rb index 682e1f5..b32fcb6 100644 --- a/lib/exceptionist.rb +++ b/lib/exceptionist.rb @@ -6,7 +6,7 @@ require 'es_client' module Exceptionist - attr_accessor :esclient, :endpoint + attr_accessor :esclient def self.esclient @esclient ||= ESClient.new(@endpoint) From ddfbee45c339ca1f73ea17b8cc14ff02c71761f0 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 13:02:31 +0200 Subject: [PATCH 024/293] remove duplication of code --- lib/tools.rb | 75 ---------------------------------------------------- 1 file changed, 75 deletions(-) diff --git a/lib/tools.rb b/lib/tools.rb index 288815e..5bdfb57 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -7,79 +7,4 @@ def self.run(uber_key) UberException.find(uber_key).forget! end end - - class Exporter - def self.run - occurrences = Occurrence.find_all.map { |occurrence| occurrence.to_hash } - - File.open('occurrences_export.json', 'w') do |file| - file.write(Yajl::Encoder.encode(occurrences)) - end - end - end - - class Importer - def self.run - files = Dir.glob('test/fixtures/occurrences_export*').sort - files.each do |file| - puts "importing #{file}" - - occurrences = Yajl::Parser.parse(File.read(file)) - occurrences.each do |occurrence_hash| - - # TODO: still problems with es mapping when indexing new documents - replace_empty_deep!(occurrence_hash) - occurrence_hash.delete('uber_key') - occurrence_hash.delete('id') - occurrence_hash['parameters'].delete('utm_source') if occurrence_hash['parameters'] - occurrence_hash['parameters'].delete('status') if occurrence_hash['parameters'] - - occurrence = Occurrence.new(occurrence_hash) - occurrence.save - - UberException.occurred(occurrence) - end - end - end - - def self.replace_empty_deep!(h) - h.each do | k, v | - - # TODO: remove custom date from date fields - if v && v == 'custom-date' - h[k] = nil - end - - if v && v.empty? - h[k] = nil - else - replace_empty_deep!(v) if v.kind_of?(Hash) - end - end - end - end - - class ClearDB - def self.run - begin - Exceptionist.esclient.delete_indices('exceptionist') - rescue Elasticsearch::Transport::Transport::Errors::NotFound - end - - Exceptionist.esclient.create_indices('exceptionist', - { mappings: { - occurrences: { properties: { - action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - } }, - exceptions:{ properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - } } - } }) - Exceptionist.esclient.refresh - end - end end From 9ce184c55c2e8104ec36253ac82470c3886184b7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 13:03:05 +0200 Subject: [PATCH 025/293] add attr host and port --- lib/es_client.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 4f42395..86212e1 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -1,12 +1,13 @@ require 'elasticsearch' class ESClient - attr_accessor :es + attr_accessor :es, :host, :port INDEX = 'exceptionist' TYPE_EXCEPTIONS = 'exceptions' TYPE_OCCURRENCES = 'occurrences' def initialize(endpoint) + @host, @port = endpoint.split(':') @es = Elasticsearch::Client.new(host: endpoint) end From 2dbe5b7912ae30ab05a1b554ebf5cecdc295195e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 14:03:52 +0200 Subject: [PATCH 026/293] remove printing of hash --- lib/es_helper.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 69fc052..ca631a2 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -34,7 +34,6 @@ def self.run occurrences = Yajl::Parser.parse(File.read(file)) occurrences.each do |occurrence_hash| - pp occurrence_hash occurrence_hash.delete('uber_key') occurrence_hash.delete('id') From f357ebbc67250e270d0c08d2df5643a19f7a5844 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 14:04:26 +0200 Subject: [PATCH 027/293] enable toggling of projects --- views/dashboard.erb | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/views/dashboard.erb b/views/dashboard.erb index 6898bd8..e4c8f33 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -1,11 +1,25 @@

<%= @title %>

+
+ <% for project in @projects %> + + <% end %> +
+ + <% for project in @projects %> -
-

<%= project.name %> (<%= project.exceptions_count %>)

+
+

<%= project.name %> (<%= project.exceptions_count %>)

-
- <%= partial(:thirty_day_graph, :holder_div_id => "holder_#{project.name}", :data => project.last_thirty_days) %> +
+ <%= partial(:thirty_day_graph, :holder_div_id => "holder_#{project.name}", :data => project.last_thirty_days) %> +
-
<% end %> + + + From b8751115e7f62161510351d3c3a541cd5a3ec0d9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 9 Sep 2014 15:00:37 +0200 Subject: [PATCH 028/293] change DB to ES in config example --- config.rb.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.rb.example b/config.rb.example index b619584..20c704d 100644 --- a/config.rb.example +++ b/config.rb.example @@ -11,8 +11,8 @@ require 'app' # ========================== # -# Configure your MongoDB server and port -Exceptionist.mongo = 'localhost:27017' +# Configure your ES server and port +Exceptionist.endpoint = 'localhost:9200' # Credentials for accessing the web app, remove if you do auth in another way Exceptionist.enable_authentication('username', 'password') From dd664577f8915e31bacbb2516ba5a15a7c50135e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 13:46:46 +0200 Subject: [PATCH 029/293] add model deploy --- lib/es_client.rb | 33 ++++++++++++++++++++++++------- lib/es_helper.rb | 9 ++++++++- lib/models/deploy.rb | 47 ++++++++++++++++++++++++++++++++++++++++++++ test/deploy_test.rb | 13 ++++++++++++ test/test_helper.rb | 18 ++++++++++++++--- 5 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 lib/models/deploy.rb create mode 100644 test/deploy_test.rb diff --git a/lib/es_client.rb b/lib/es_client.rb index 86212e1..7d111c2 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -5,6 +5,7 @@ class ESClient INDEX = 'exceptionist' TYPE_EXCEPTIONS = 'exceptions' TYPE_OCCURRENCES = 'occurrences' + TYPE_DEPLOYS = 'deploys' def initialize(endpoint) @host, @port = endpoint.split(':') @@ -84,16 +85,34 @@ def refresh end private - def create_occurrence(attributes) - return nil unless attributes + def create_occurrence(attr) + attr = transform(attr) + Occurrence.new(attr) + end + + def create_exception(attr) + attr = transform(attr) + UberException.new(attr) + end + + def create_deploy(attr) + attr = transform(attr) + Deploy.new(attr) + end - attributes.merge!(attributes['_source']).delete('_source') - Occurrence.new(attributes) + def transform(attr) + attr.merge!(attr['_source']).delete('_source') + attr = symbolize_keys(attr) + attr[:id] = attr.delete :_id + attr.delete :_index + attr.delete :_type + attr.delete :_score + attr.delete :sort + attr end - def create_exception(attributes) - attributes.merge!(attributes['_source']).delete('_source') - UberException.new(attributes) + def symbolize_keys(hash) + hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} end def create_search_query(terms, sort, from, size) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index ca631a2..671a18d 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -73,7 +73,14 @@ def self.run project_name: { type: 'string', index: 'not_analyzed' }, closed: { type: 'boolean' }, occurred_at: { type: 'date' } - } } + } }, + deploys: { + properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + version: { type: 'string', index: 'not_analyzed' }, + deploy_time: { type: 'date' }, + changelog_link: { type: 'string', index: 'not_analyzed'} + } }, } }) Exceptionist.esclient.refresh end diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb new file mode 100644 index 0000000..4bb861d --- /dev/null +++ b/lib/models/deploy.rb @@ -0,0 +1,47 @@ +require 'json' + +class Deploy + attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :deploy_time + + def self.from_json(json) + attr = symbolize_keys(JSON.parse(json)) + Deploy.new(attr) + end + + def occurred_at + deploy_time.is_a?(String) ? Time.parse(deploy_time) : deploy_time + end + + def initialize(attributes={}) + attributes.each do |key, value| + send("#{key}=", value) + end + + self.deploy_time ||= Time.now + end + + def save + deploy = Exceptionist.esclient.index('deploys', to_hash) + @id = deploy._id + self + end + + def ==(other) + project_name == other.project_name && version == other.version + end + + private + def to_hash + { + project_name: project_name, + api_key: api_key, + version: version, + changelog_link: changelog_link, + deploy_time: deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + } + end + + def self.symbolize_keys(hash) + hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} + end +end diff --git a/test/deploy_test.rb b/test/deploy_test.rb new file mode 100644 index 0000000..20cb59a --- /dev/null +++ b/test/deploy_test.rb @@ -0,0 +1,13 @@ +require 'test_helper' + +class DeployTest < AbstractTest + + def test_save + deploy = create_deploy + + Exceptionist.esclient.refresh + + assert_equal deploy, Deploy.find_all('ExampleProject').first + assert_equal 0, Deploy.find_all('OtherProject').count + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 3fd45ff..053223c 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,8 +1,6 @@ require 'es_helper' require 'app' -port = 10000 - at_exit do ESHelper.stopCluster end @@ -10,7 +8,7 @@ # minitest install its own at_exit, so we need to do this after our own require 'minitest/autorun' -Exceptionist.endpoint = "localhost:#{port}" +Exceptionist.endpoint = "localhost:10000" ESHelper.startCluster ESHelper::ClearDB.run @@ -42,6 +40,20 @@ def create_occurrence(attributes = {}) build_occurrence(attributes).save end +def build_deploy(attributes = {}) + default_attributes = { + project_name: 'ExampleProject', + api_key: 'SECRET_API_KEY', + version: '0.0.1', + changelog_link: 'https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e' + } + Deploy.new(default_attributes.merge(attributes)) +end + +def create_deploy(attributes = {}) + build_deploy(attributes).save +end + def clear_collections Exceptionist.esclient.delete_by_query( match_all: {}) end From 63326c39bbfb291bf4fa196d4d1ca56c5bae6eb3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 13:57:30 +0200 Subject: [PATCH 030/293] update variables --- lib/models/occurrence.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 99a8d0d..4dba924 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -2,8 +2,7 @@ class Occurrence attr_accessor :url, :controller_name, :action_name, :exception_class, :exception_message, :exception_backtrace, :parameters, :session, :cgi_data, :environment, - :project_name, :occurred_at, :occurred_at_day, :_id, :uber_key, :api_key, - :_score, :sort + :project_name, :occurred_at, :occurred_at_day, :id, :uber_key, :api_key, :sort def initialize(attributes={}) From e0add32e6e36c00da6cc1b078e8a3c7dcf4e9549 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 13:58:40 +0200 Subject: [PATCH 031/293] es specific variables are already dropped --- lib/models/occurrence.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 4dba924..cb613ea 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -7,7 +7,7 @@ class Occurrence def initialize(attributes={}) attributes.each do |key, value| - send("#{key}=", value) unless ["_index", "_type"].include?(key) + send("#{key}=", value) end self.occurred_at ||= attributes['occurred_at'] || Time.now From ed6f1f9d66df28ad9ccfe593541e7f0c09200c5f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 13:59:56 +0200 Subject: [PATCH 032/293] rename _id to id --- lib/models/occurrence.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index cb613ea..813aa7a 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -15,11 +15,11 @@ def initialize(attributes={}) end def inspect - "(Occurrence: id: #{_id}, title: '#{title}')" + "(Occurrence: id: #{id}, title: '#{title}')" end def ==(other) - _id == other._id + id == other.id end def title @@ -87,7 +87,7 @@ def self.find_all_by_name(project, limit=50) def save occurrence = Exceptionist.esclient.index('occurrences', to_hash) - @_id = occurrence._id + @id = occurrence._id self end From 4bccc0bd2321c4ee6aaec3063f320533dd5afacc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:01:05 +0200 Subject: [PATCH 033/293] use new hash syntax --- lib/models/occurrence.rb | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 813aa7a..f0310e7 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -103,20 +103,20 @@ def self.es_create(attributes) end def to_hash - { :exception_message => exception_message, - :session => session, - :action_name => action_name, - :parameters => parameters, - :cgi_data => cgi_data, - :url => url, - :occurred_at => occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), - :occurred_at_day => occurred_at.strftime('%Y-%m-%d'), - :exception_backtrace => exception_backtrace, - :controller_name => controller_name, - :environment => environment, - :exception_class => exception_class, - :project_name => project_name, - :uber_key => uber_key } + { exception_message: exception_message, + session: session, + action_name: action_name, + parameters: parameters, + cgi_data: cgi_data, + url: url, + occurred_at: occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), + occurred_at_day: occurred_at.strftime('%Y-%m-%d'), + exception_backtrace: exception_backtrace, + controller_name: controller_name, + environment: environment, + exception_class: exception_class, + project_name: project_name, + uber_key: uber_key } end def self.from_xml(xml_text) From c282a74ba1fc323d742550fb8fda64cd45288437 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:02:18 +0200 Subject: [PATCH 034/293] right indention --- lib/models/occurrence.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index f0310e7..ea9bcdd 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -178,19 +178,19 @@ def self.parse_optional_element(doc, xpath) element ? element.content : nil end -private + private def generate_uber_key key = case exception_class - when *Exceptionist.global_exception_classes - "#{exception_class}:#{exception_message}" - when *Exceptionist.timeout_exception_classes - first_non_lib_line = exception_backtrace.detect { |line| line =~ /\[PROJECT_ROOT\]/ } - "#{exception_class}:#{exception_message}:#{first_non_lib_line}" - else - backtrace = exception_backtrace ? exception_backtrace.first : '' - "#{controller_name}:#{action_name}:#{exception_class}:#{backtrace}" - end + when *Exceptionist.global_exception_classes + "#{exception_class}:#{exception_message}" + when *Exceptionist.timeout_exception_classes + first_non_lib_line = exception_backtrace.detect { |line| line =~ /\[PROJECT_ROOT\]/ } + "#{exception_class}:#{exception_message}:#{first_non_lib_line}" + else + backtrace = exception_backtrace ? exception_backtrace.first : '' + "#{controller_name}:#{action_name}:#{exception_class}:#{backtrace}" + end Digest::SHA1.hexdigest("#{project_name}:#{key}") end From 6a7614b350fa3abd4015d6502fb22b52067eeec8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:03:38 +0200 Subject: [PATCH 035/293] add model deploy --- lib/boot.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/boot.rb b/lib/boot.rb index 8a4be74..c922ce5 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -10,6 +10,7 @@ require 'models/project' require 'models/uber_exception' require 'models/occurrence' +require 'models/deploy' require 'models/mailer' require 'exceptionist' From 866b8ad2ca3748f7d68bd6698d7dd67310e8cb0b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:04:45 +0200 Subject: [PATCH 036/293] use symbol instead of strings --- lib/models/uber_exception.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 6fa3179..178694b 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -2,9 +2,10 @@ class UberException attr_accessor :id, :project_name, :occurrences_count, :closed def initialize(attributes) - @id = attributes['_id'] - @project_name = attributes['project_name'] - @occurrences_count = attributes['occurrences_count'] + @id = attributes[:id] + @project_name = attributes[:project_name] + @occurrences_count = attributes[:occurrences_count] + @closed = attributes[:closed] end def self.create_es(attributes) From fdaf0dd5bcd78da6196c91c6ee6a0a4379507bb1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:05:46 +0200 Subject: [PATCH 037/293] query for new object and return it --- lib/models/uber_exception.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 178694b..f89ebe9 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -47,9 +47,7 @@ def self.find_new_on(project, day) def self.occurred(occurrence) Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1', upsert: { project_name: occurrence.project_name, occurred_at: occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), closed: false, occurrences_count: 1 } }) - - # return the UberException - new('_id' => occurrence.uber_key) + Exceptionist.esclient.get_exception(occurrence.uber_key) end def self.forget_old_exceptions(project, days) From 0a06c0cdf50298e22534e9a9d263cebd88e8089f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:06:08 +0200 Subject: [PATCH 038/293] use method from occurrence class --- lib/models/uber_exception.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index f89ebe9..b57f115 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -115,7 +115,7 @@ def url end def occurrences_count_on(date) - Exceptionist.esclient.count('occurrences', [ { term: { uber_key: id } }, term: { occurred_at_day: date.strftime('%Y-%m-%d') } ]) + Occurrence.count_all_on(project_name, date) end def last_thirty_days From 4baad525ff2bd3b8497a6d5a3e6a40a7244222e2 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:08:00 +0200 Subject: [PATCH 039/293] add find all to deploy model --- lib/es_client.rb | 7 +++++++ lib/models/deploy.rb | 4 ++++ test/deploy_test.rb | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/lib/es_client.rb b/lib/es_client.rb index 7d111c2..4c40b4e 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -26,6 +26,13 @@ def search_exceptions(filters, sort={}, from=0, size=50) hash.hits.hits.map { |doc| create_exception(doc) } end + def search_deploys(filters, sort={}, from=0, size=50) + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: TYPE_DEPLOYS, body: query) + hash = Hashie::Mash.new(response) + hash.hits.hits.map { |doc| create_deploy(doc) } + end + def search_aggs(filters, aggs) query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs } } } } response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 4bb861d..2a83a7e 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -3,6 +3,10 @@ class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :deploy_time + def self.find_all(project) + Exceptionist.esclient.search_deploys( term: { project_name: project } ) + end + def self.from_json(json) attr = symbolize_keys(JSON.parse(json)) Deploy.new(attr) diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 20cb59a..31664ab 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -10,4 +10,15 @@ def test_save assert_equal deploy, Deploy.find_all('ExampleProject').first assert_equal 0, Deploy.find_all('OtherProject').count end + + def test_find_all + create_deploy + create_deploy( version: '0.0.2' ) + create_deploy( project_name: 'OtherProject' ) + + Exceptionist.esclient.refresh + + assert_equal 2, Deploy.find_all('ExampleProject').count + assert_equal 1, Deploy.find_all('OtherProject').count + end end From 612f02dc10661112d50a0bebc63bd66269f0dec8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:09:23 +0200 Subject: [PATCH 040/293] add find last deploy --- lib/models/deploy.rb | 4 ++++ test/deploy_test.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 2a83a7e..a32a4b9 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -7,6 +7,10 @@ def self.find_all(project) Exceptionist.esclient.search_deploys( term: { project_name: project } ) end + def self.find_last_deploy(project) + Exceptionist.esclient.search_deploys( { term: { project_name: project } }, { deploy_time: { order: 'desc' } }, from: 0, size: 1 ).first + end + def self.from_json(json) attr = symbolize_keys(JSON.parse(json)) Deploy.new(attr) diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 31664ab..44552c8 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -21,4 +21,17 @@ def test_find_all assert_equal 2, Deploy.find_all('ExampleProject').count assert_equal 1, Deploy.find_all('OtherProject').count end + + def test_find_last_deploy + assert_nil Deploy.find_last_deploy('ExampleProject') + + create_deploy + deploy = create_deploy( version: '0.0.2' ) + deploy_other_project = create_deploy( project_name: 'OtherProject' ) + + Exceptionist.esclient.refresh + + assert_equal deploy, Deploy.find_last_deploy('ExampleProject') + assert_equal deploy_other_project, Deploy.find_last_deploy('OtherProject') + end end From 7a83ec2a13876a12b0377baa8c6cf4183f767ddd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:10:35 +0200 Subject: [PATCH 041/293] add api endpoint for deploy --- lib/app.rb | 14 ++++++++++++++ test/api_test.rb | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/lib/app.rb b/lib/app.rb index d05b50c..7cc78de 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -121,6 +121,20 @@ class ExceptionistApp < Sinatra::Base end end + post '/notifier_api/v2/deploy/?' do + deploy = Deploy.from_json(params[:data] || request.body.read) + project = Project.find_by_key(deploy.api_key) + if project + deploy.project_name = project.name + deploy.save + + "#{deploy.id}" + else + status 401 + 'Invalid API Key' + end + end + helpers do include Rack::Utils diff --git a/test/api_test.rb b/test/api_test.rb index 38c7571..ef6464e 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -89,4 +89,15 @@ def test_api_minimal_exception assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end + + def test_api_deploy + assert_equal [], Deploy.find_all('ExampleProject') + + post '/notifier_api/v2/deploy/', read_fixtures_file('fixtures/deploy.json') + assert last_response.ok? + + Exceptionist.esclient.refresh + + assert_equal 1, Deploy.find_all('ExampleProject').count + end end From ee53d0bcae6f0087105b6aeda0d8c7809e41190a Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:11:31 +0200 Subject: [PATCH 042/293] add last deploy function to project --- lib/models/project.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/models/project.rb b/lib/models/project.rb index 8d3de74..8019d68 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -25,6 +25,10 @@ def self.last_n_days(days) n_days end + def last_deploy + Deploy.find_last_deploy(name) + end + def occurrence_count_on(date) Occurrence.count_all_on(name, date) end From 47f5fc0b48e47badff9d7ded3afa48871d87b1ee Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:12:38 +0200 Subject: [PATCH 043/293] add last deploy date to dashboard --- test/integration_test.rb | 21 +++++++++++++++++++++ views/dashboard.erb | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index d46ace1..58b75f0 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -32,6 +32,27 @@ def test_show_the_dashboard_with_one_project click_link 'ExampleProject' end + def test_show_the_dashboard_with_no_deploy + occurrence = create_occurrence + UberException.occurred(occurrence) + + Exceptionist.esclient.refresh + + visit '/' + assert_contain '- no deploy found' + end + + def test_show_the_dashboard_with_deploy + occurrence = create_occurrence + UberException.occurred(occurrence) + create_deploy + + Exceptionist.esclient.refresh + + visit '/' + assert_contain '- deploy:' + end + def test_show_the_dashboard_with_two_projects occur1 = create_occurrence(project_name: 'ExampleProject') UberException.occurred(occur1) diff --git a/views/dashboard.erb b/views/dashboard.erb index e4c8f33..b4ce42f 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -9,7 +9,8 @@ <% for project in @projects %>
-

<%= project.name %> (<%= project.exceptions_count %>)

+

<%= project.name %> (<%= project.exceptions_count %>) + <% if project.last_deploy %> - deploy: <%= format_time(project.last_deploy.occurred_at) %> <% else %> - no deploy found in db :(<% end %>

<%= partial(:thirty_day_graph, :holder_div_id => "holder_#{project.name}", :data => project.last_thirty_days) %> From a8ec98666c71aa0a3c1ab94fdcd1c842342e716e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 14:47:03 +0200 Subject: [PATCH 044/293] check if deploy time is string or time --- lib/models/deploy.rb | 2 +- test/deploy_test.rb | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index a32a4b9..d1939ed 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -45,7 +45,7 @@ def to_hash api_key: api_key, version: version, changelog_link: changelog_link, - deploy_time: deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + deploy_time: deploy_time.is_a?(String) ? deploy_time : deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } end diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 44552c8..2bf34d1 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -11,6 +11,15 @@ def test_save assert_equal 0, Deploy.find_all('OtherProject').count end + def test_save_with_deploy_time + deploy = create_deploy( deploy_time: '2014-09-10T14:45:42.125+0200' ) + + Exceptionist.esclient.refresh + + assert_equal deploy, Deploy.find_all('ExampleProject').first + assert_equal 0, Deploy.find_all('OtherProject').count + end + def test_find_all create_deploy create_deploy( version: '0.0.2' ) From 0de4d8988dfdddd92a3f4de2e341c14d8f489201 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 15:39:48 +0200 Subject: [PATCH 045/293] add occurrences count to mapping --- lib/es_helper.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 671a18d..99dfc57 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -72,7 +72,8 @@ def self.run properties: { project_name: { type: 'string', index: 'not_analyzed' }, closed: { type: 'boolean' }, - occurred_at: { type: 'date' } + occurred_at: { type: 'date' }, + occurrences_count: {type: 'long'} } }, deploys: { properties: { From 93e7fecb51430df0253a593d90c2fbc3a8986d25 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 10 Sep 2014 15:40:13 +0200 Subject: [PATCH 046/293] improve orderd by occurrence count test --- test/uber_exception_test.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 959b842..f177d39 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -51,12 +51,16 @@ def test_find_all_sorted_by_time def test_find_all_sorted_by_occurrences_count exce1 = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) + UberException.occurred(create_occurrence()) + exce3 = UberException.occurred(create_occurrence(action_name: 'different')) exce2 = UberException.occurred(create_occurrence(action_name: 'other')) + UberException.occurred(create_occurrence(action_name: 'other')) + Exceptionist.esclient.refresh - assert_equal [exce1, exce2], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 0, 10) - assert_equal [exce2], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 1, 10) + assert_equal [exce1, exce2, exce3], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 0, 10) + assert_equal [exce2, exce3], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 1, 10) end def test_find_new_on From 6c9a71a423613d646246854b8f9fcd149ce5d9f9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 09:48:25 +0200 Subject: [PATCH 047/293] remove inspect function for easier debugging --- lib/models/uber_exception.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index b57f115..1e1475e 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -126,7 +126,4 @@ def ==(other) id == other.id end - def inspect - "(UberException: id: #{id})" - end end From 20c50fbca3f694a613729b7fdf511d810b60cf68 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 09:50:52 +0200 Subject: [PATCH 048/293] use keyword arguments for occurrence search --- lib/es_client.rb | 4 ++-- lib/models/occurrence.rb | 21 ++++++++++++++++----- lib/models/uber_exception.rb | 9 +-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 4c40b4e..5ceb43a 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -12,9 +12,9 @@ def initialize(endpoint) @es = Elasticsearch::Client.new(host: endpoint) end - def search_occurrences(filters, sort={}, from=0, size=50) + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 50) query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) + response = @es.search(index: INDEX, type: type, body: query) hash = Hashie::Mash.new(response) hash.hits.hits.map { |doc| create_occurrence(doc) } end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index ea9bcdd..5623910 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -60,12 +60,12 @@ def self.delete_all_for(uber_key) end def self.find_first_for(uber_key) - occurrences = Exceptionist.esclient.search_occurrences( { term: { uber_key: uber_key } }, { occurred_at: { order: 'asc' } }, from: 0, size: 1 ) + occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'asc' } }, size: 1 ) occurrences.first end def self.find_last_for(uber_key) - occurrences = Exceptionist.esclient.search_occurrences( { term: { uber_key: uber_key } }, { occurred_at: { order: 'desc' } }, from: 0, size: 1 ) + occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'desc' } }, size: 1 ) occurrences.first end @@ -74,11 +74,22 @@ def self.count_all_on(project, day) end def self.find_all(size=50) - Exceptionist.esclient.search_occurrences( {}, { occurred_at: { order: 'desc' } }, 0, size ) + Exceptionist.esclient.search( sort: { occurred_at: { order: 'desc' } }, size: size ) end - def self.find_all_by_name(project, limit=50) - Exceptionist.esclient.search_occurrences( { term: { project_name: project } }, { occurred_at: { order: 'desc' } }, from: 0, size: limit ) + def self.find_all_by_name(project, size=50) + Exceptionist.esclient.search( filters: { term: { project_name: project } }, sort: { occurred_at: { order: 'desc' } }, size: size ) + end + + def self.get_occurrence(uber_key: '', position: 1) + return nil if position < 1 + + response = Exceptionist.esclient.search(filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) + if response.any? + response.first + else + nil + end end # diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 1e1475e..07ec911 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -83,14 +83,7 @@ def first_occurrence end def current_occurrence(position) - return nil if position < 1 - - response = Exceptionist.esclient.search_occurrences({ term: { uber_key: id } }, { occurred_at: { order: 'asc'} }, position - 1, 1) - if response.any? - response.first - else - nil - end + Occurrence.get_occurrence(uber_key: id, position: position) end def update_occurrences_count From 4a6fa82f0f1f19c4bd5483e5e43659372ff0c45c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 10:17:02 +0200 Subject: [PATCH 049/293] rename occurred_at to last_occurred_at --- lib/es_helper.rb | 2 +- lib/models/uber_exception.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 99dfc57..d6f53b6 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -72,7 +72,7 @@ def self.run properties: { project_name: { type: 'string', index: 'not_analyzed' }, closed: { type: 'boolean' }, - occurred_at: { type: 'date' }, + last_occurred_at: { type: 'date' }, occurrences_count: {type: 'long'} } }, deploys: { diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 07ec911..a748b81 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,5 +1,5 @@ class UberException - attr_accessor :id, :project_name, :occurrences_count, :closed + attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurred_at def initialize(attributes) @id = attributes[:id] @@ -26,7 +26,7 @@ def self.find_all(project) end def self.find_all_sorted_by_time(project, start, limit) - Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { occurred_at: { order: 'desc'} }, start, limit) + Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { last_occurred_at: { order: 'desc'} }, start, limit) end def self.find_all_sorted_by_occurrences_count(project, start, limit) @@ -46,7 +46,7 @@ def self.find_new_on(project, day) def self.occurred(occurrence) Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1', upsert: - { project_name: occurrence.project_name, occurred_at: occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), closed: false, occurrences_count: 1 } }) + { project_name: occurrence.project_name, last_occurred_at: occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), closed: false, occurrences_count: 1 } }) Exceptionist.esclient.get_exception(occurrence.uber_key) end @@ -54,7 +54,7 @@ def self.forget_old_exceptions(project, days) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.esclient.search_exceptions( [ { term: { project_name: project } }, range: { occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + uber_exceptions = Exceptionist.esclient.search_exceptions( [ { term: { project_name: project } }, range: { last_occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) uber_exceptions.each do |exception| exception.forget! From e401fb07489dabebb18d2f557f3f65d9d8665af0 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 10:42:55 +0200 Subject: [PATCH 050/293] reuse find method from occurrence model --- lib/models/occurrence.rb | 25 ++++++++++--------------- lib/models/uber_exception.rb | 2 +- test/uber_exception_test.rb | 5 ++++- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 5623910..20867d0 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -60,13 +60,19 @@ def self.delete_all_for(uber_key) end def self.find_first_for(uber_key) - occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'asc' } }, size: 1 ) - occurrences.first + Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'asc' } }) end def self.find_last_for(uber_key) - occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'desc' } }, size: 1 ) - occurrences.first + Occurrence.find(uber_key: uber_key) + end + + def self.find(uber_key: '', sort: { occurred_at: { order: 'desc' } }, position: 0) + raise ArgumentError, 'position has to be >= 0' if position < 0 + + occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: 1 ) + + occurrences.any? ? occurrences.first : nil end def self.count_all_on(project, day) @@ -81,17 +87,6 @@ def self.find_all_by_name(project, size=50) Exceptionist.esclient.search( filters: { term: { project_name: project } }, sort: { occurred_at: { order: 'desc' } }, size: size ) end - def self.get_occurrence(uber_key: '', position: 1) - return nil if position < 1 - - response = Exceptionist.esclient.search(filters: { term: { uber_key: uber_key } }, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) - if response.any? - response.first - else - nil - end - end - # # serialization # diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index a748b81..9687f47 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -83,7 +83,7 @@ def first_occurrence end def current_occurrence(position) - Occurrence.get_occurrence(uber_key: id, position: position) + Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, position: position - 1) end def update_occurrences_count diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index f177d39..065d812 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -154,8 +154,11 @@ def test_current_occurrence assert_equal ocr1, uber_ex.current_occurrence(1) assert_equal ocr2, uber_ex.current_occurrence(2) - assert_equal nil, uber_ex.current_occurrence(0) assert_equal nil, uber_ex.current_occurrence(3) + + assert_raises ArgumentError do + uber_ex.current_occurrence(0) + end end def test_update_occurence_count From 55e686f3339cd0f28b4bdbfb98d41ce6f450e94c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 10:49:53 +0200 Subject: [PATCH 051/293] refactor delete by query func to use keyword arg --- lib/es_client.rb | 2 +- lib/models/occurrence.rb | 2 +- test/test_helper.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 5ceb43a..f9bd47e 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -62,7 +62,7 @@ def count(type, terms) @es.count(index: INDEX, type: type, body: query)['count'] end - def delete_by_query(query) + def delete_by_query(query: { match_all: {} }) @es.delete_by_query( index: INDEX, body: { query: query } ) end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 20867d0..4e901b7 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -56,7 +56,7 @@ def uber_exception end def self.delete_all_for(uber_key) - Exceptionist.esclient.delete_by_query({ term: { uber_key: uber_key } }) + Exceptionist.esclient.delete_by_query(query: { term: { uber_key: uber_key } }) end def self.find_first_for(uber_key) diff --git a/test/test_helper.rb b/test/test_helper.rb index 053223c..4b96040 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -55,7 +55,7 @@ def create_deploy(attributes = {}) end def clear_collections - Exceptionist.esclient.delete_by_query( match_all: {}) + Exceptionist.esclient.delete_by_query end class AbstractTest < Minitest::Test From 7df78d8ac9b14efa44bbc7a24e20c912c0c7e58d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 11:04:03 +0200 Subject: [PATCH 052/293] refactor count func to use keyword args --- lib/es_client.rb | 2 +- lib/models/occurrence.rb | 2 +- lib/models/uber_exception.rb | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index f9bd47e..fae489c 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -55,7 +55,7 @@ def update(type, id, body) @es.update(index: INDEX, type: type, id: id, body: body) end - def count(type, terms) + def count(type: 'occurrences', terms: []) terms = [terms] if terms.class == Hash query = { query: { filtered: { filter: { bool: { must: terms } } } } } diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 4e901b7..f8ba691 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -76,7 +76,7 @@ def self.find(uber_key: '', sort: { occurred_at: { order: 'desc' } }, position: end def self.count_all_on(project, day) - Exceptionist.esclient.count('occurrences', [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) + Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) end def self.find_all(size=50) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 9687f47..00ed8a4 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -14,7 +14,7 @@ def self.create_es(attributes) end def self.count_all(project) - Exceptionist.esclient.count('exceptions', { term: { project_name: project } } ) + Exceptionist.esclient.count(type: 'exceptions', terms: { term: { project_name: project } } ) end def self.find(uber_key) @@ -87,7 +87,7 @@ def current_occurrence(position) end def update_occurrences_count - @occurrences_count = Exceptionist.esclient.count('occurrences', { term: { uber_key: id } }) + @occurrences_count = Exceptionist.esclient.count(terms: { term: { uber_key: id } }) Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: @occurrences_count } }) end From 7aba7ac749152cacdc2dc0e90373734e4428a7b6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 11:31:12 +0200 Subject: [PATCH 053/293] refactor search in es client to use keyword args --- lib/es_client.rb | 18 ++++++++++-------- lib/models/occurrence.rb | 6 +++--- lib/models/uber_exception.rb | 20 ++++++++++++++------ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index fae489c..d4f1ba2 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -12,20 +12,22 @@ def initialize(endpoint) @es = Elasticsearch::Client.new(host: endpoint) end - def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 50) - query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: type, body: query) - hash = Hashie::Mash.new(response) + def search_occurrences(filters: {}, sort: {}, from: 0, size: 50) + hash = search(filters: filters, sort: sort, from: from, size: size) hash.hits.hits.map { |doc| create_occurrence(doc) } end - def search_exceptions(filters, sort={}, from=0, size=50) - query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: TYPE_EXCEPTIONS, body: query) - hash = Hashie::Mash.new(response) + def search_exceptions(filters: {}, sort: {}, from: 0, size: 50) + hash = search(type: TYPE_EXCEPTIONS, filters: filters, sort: sort, from: from, size: size) hash.hits.hits.map { |doc| create_exception(doc) } end + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 50) + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: type, body: query) + Hashie::Mash.new(response) + end + def search_deploys(filters, sort={}, from=0, size=50) query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: TYPE_DEPLOYS, body: query) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index f8ba691..4bea986 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -70,7 +70,7 @@ def self.find_last_for(uber_key) def self.find(uber_key: '', sort: { occurred_at: { order: 'desc' } }, position: 0) raise ArgumentError, 'position has to be >= 0' if position < 0 - occurrences = Exceptionist.esclient.search( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: 1 ) + occurrences = Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: 1 ) occurrences.any? ? occurrences.first : nil end @@ -80,11 +80,11 @@ def self.count_all_on(project, day) end def self.find_all(size=50) - Exceptionist.esclient.search( sort: { occurred_at: { order: 'desc' } }, size: size ) + Exceptionist.esclient.search_occurrences( sort: { occurred_at: { order: 'desc' } }, size: size ) end def self.find_all_by_name(project, size=50) - Exceptionist.esclient.search( filters: { term: { project_name: project } }, sort: { occurred_at: { order: 'desc' } }, size: size ) + Exceptionist.esclient.search_occurrences( filters: { term: { project_name: project } }, sort: { occurred_at: { order: 'desc' } }, size: size ) end # diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 00ed8a4..df292aa 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -22,15 +22,23 @@ def self.find(uber_key) end def self.find_all(project) - Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ]) + Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ]) end - def self.find_all_sorted_by_time(project, start, limit) - Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { last_occurred_at: { order: 'desc'} }, start, limit) + def self.find_all_sorted_by_time(project, from, size) + Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ], sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) end - def self.find_all_sorted_by_occurrences_count(project, start, limit) - Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } } ], { occurrences_count: { order: 'desc' } }, start, limit ) + def self.find_all_sorted_by_time_since(project, since, from, size) + Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } }, range: { last_occurred_at: { gte: since.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ], sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) + end + + def self.find_all_sorted_by_occurrences_count(project, from, size) + Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ], sort: { occurrences_count: { order: 'desc' } }, from: from, size: size ) + end + + def self.find_all_sorted_by_occurrences_count_since(project, since, from, size) + Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } }, range: { last_occurred_at: { gte: since.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ], { occurrences_count: { order: 'desc'} }, from: from, size: size) end def self.find_new_on(project, day) @@ -54,7 +62,7 @@ def self.forget_old_exceptions(project, days) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.esclient.search_exceptions( [ { term: { project_name: project } }, range: { last_occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { last_occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) uber_exceptions.each do |exception| exception.forget! From 34ea7db700a047740befc3a76ef6741b3bd3ff13 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 12:26:17 +0200 Subject: [PATCH 054/293] rename function find to get --- lib/app.rb | 6 +++--- lib/models/occurrence.rb | 2 +- lib/models/uber_exception.rb | 2 +- lib/tools.rb | 2 +- test/uber_exception_test.rb | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 7cc78de..e47ec92 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -74,7 +74,7 @@ class ExceptionistApp < Sinatra::Base get '/exceptions/:id' do @projects = Project.all - @uber_exception = UberException.find(params[:id]) + @uber_exception = UberException.get(params[:id]) @occurrence_position = @uber_exception.occurrences_count @occurrence = @uber_exception.current_occurrence(@occurrence_position) @@ -91,7 +91,7 @@ class ExceptionistApp < Sinatra::Base end get '/exceptions/:id/occurrences/:occurrence_position' do - @uber_exception = UberException.find(params[:id]) + @uber_exception = UberException.get(params[:id]) @occurrence_position = params[:occurrence_position].to_i @occurrence = @uber_exception.current_occurrence(@occurrence_position) @@ -99,7 +99,7 @@ class ExceptionistApp < Sinatra::Base end post '/exceptions/:id/close' do - @uber_exceptions = UberException.find(params[:id]) + @uber_exceptions = UberException.get(params[:id]) @uber_exceptions.close! redirect to("/projects/#{@uber_exceptions.project_name}?#{Rack::Utils.unescape(params[:backparams])}") diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 4bea986..a2cce1b 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -52,7 +52,7 @@ def project end def uber_exception - UberException.find(uber_key) + UberException.get(uber_key) end def self.delete_all_for(uber_key) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index df292aa..83130ae 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -17,7 +17,7 @@ def self.count_all(project) Exceptionist.esclient.count(type: 'exceptions', terms: { term: { project_name: project } } ) end - def self.find(uber_key) + def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end diff --git a/lib/tools.rb b/lib/tools.rb index 5bdfb57..9d5877a 100644 --- a/lib/tools.rb +++ b/lib/tools.rb @@ -4,7 +4,7 @@ module Exceptionist class Remover def self.run(uber_key) - UberException.find(uber_key).forget! + UberException.get(uber_key).forget! end end end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 065d812..b5c4b7f 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -22,7 +22,7 @@ def test_find Exceptionist.esclient.refresh - assert_equal uber_exception.id, UberException.find(uber_exception.id).id + assert_equal uber_exception.id, UberException.get(uber_exception.id).id end def test_find_all @@ -130,7 +130,7 @@ def test_forget! Exceptionist.esclient.refresh assert_raises(Elasticsearch::Transport::Transport::Errors::NotFound) do - UberException.find(uber_exce.id) + UberException.get(uber_exce.id) end end From 226f042c929855da934f8aa80fd22917913611c8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 12:51:35 +0200 Subject: [PATCH 055/293] remove all from find method --- lib/models/occurrence.rb | 10 +++++----- lib/models/project.rb | 4 ++-- lib/models/uber_exception.rb | 22 ++++++++-------------- test/api_test.rb | 22 +++++++++++----------- test/uber_exception_test.rb | 28 ++++++++++++++-------------- 5 files changed, 40 insertions(+), 46 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index a2cce1b..e631466 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -64,10 +64,10 @@ def self.find_first_for(uber_key) end def self.find_last_for(uber_key) - Occurrence.find(uber_key: uber_key) + Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'desc' } }) end - def self.find(uber_key: '', sort: { occurred_at: { order: 'desc' } }, position: 0) + def self.find(uber_key: '', sort: {}, position: 0) raise ArgumentError, 'position has to be >= 0' if position < 0 occurrences = Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: 1 ) @@ -79,12 +79,12 @@ def self.count_all_on(project, day) Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) end - def self.find_all(size=50) - Exceptionist.esclient.search_occurrences( sort: { occurred_at: { order: 'desc' } }, size: size ) + def self.find_all(filters: {}, sort: { occurred_at: { order: 'desc' } }, size: 50) + Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, size: size ) end def self.find_all_by_name(project, size=50) - Exceptionist.esclient.search_occurrences( filters: { term: { project_name: project } }, sort: { occurred_at: { order: 'desc' } }, size: size ) + Occurrence.find_all(filters: { term: { project_name: project } }, size: size) end # diff --git a/lib/models/project.rb b/lib/models/project.rb index 8019d68..5dc14a3 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -34,11 +34,11 @@ def occurrence_count_on(date) end def latest_exceptions(start, limit = 25) - UberException.find_all_sorted_by_time(name, start, limit) + UberException.find_sorted_by_time(name, start, limit) end def most_frequest_exceptions(start, limit = 25) - UberException.find_all_sorted_by_occurrences_count(name, start, limit) + UberException.find_sorted_by_occurrences_count(name, start, limit) end def new_exceptions_on(day) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 83130ae..6182175 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -21,24 +21,18 @@ def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end - def self.find_all(project) - Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ]) + def self.find_sorted_by_occurrences_count(project, from, size) + UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end - def self.find_all_sorted_by_time(project, from, size) - Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ], sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) + def self.find_sorted_by_time(project, from, size) + UberException.find(project: project, sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) end - def self.find_all_sorted_by_time_since(project, since, from, size) - Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } }, range: { last_occurred_at: { gte: since.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ], sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) - end - - def self.find_all_sorted_by_occurrences_count(project, from, size) - Exceptionist.esclient.search_exceptions(filters: [ { term: { project_name: project } }, { term: { closed: false } } ], sort: { occurrences_count: { order: 'desc' } }, from: from, size: size ) - end - - def self.find_all_sorted_by_occurrences_count_since(project, since, from, size) - Exceptionist.esclient.search_exceptions([ { term: { project_name: project } }, { term: { closed: false } }, range: { last_occurred_at: { gte: since.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ], { occurrences_count: { order: 'desc'} }, from: from, size: size) + def self.find(project: '', filters: [], sort: {}, from: 0, size: 50) + filters = [filters] if filters.class == Hash + filters << { term: { closed: false } } << { term: { project_name: project } } + Exceptionist.esclient.search_exceptions(filters: filters, sort: sort, from: from, size: size) end def self.find_new_on(project, day) diff --git a/test/api_test.rb b/test/api_test.rb index ef6464e..cb9534f 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -10,14 +10,14 @@ def app end def test_create_the_first_UberException - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find( project: 'ExampleProject' ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find_all('ExampleProject') + exce = UberException.find( project: 'ExampleProject' ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end @@ -28,14 +28,14 @@ def test_add_occurrences_if_it_is_the_same_exception Exceptionist.esclient.refresh - assert_equal 1, UberException.find_all('ExampleProject').count + assert_equal 1, UberException.find( project: 'ExampleProject' ).count post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find_all('ExampleProject') + exce = UberException.find( project: 'ExampleProject' ) assert_equal 1, exce.count assert_equal 2, exce.first.occurrences_count end @@ -48,44 +48,44 @@ def test_check_if_api_key_is_valid Exceptionist.esclient.refresh - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find( project: 'ExampleProject' ) end def test_api_full_exception - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find( project: 'ExampleProject' ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/full_exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find_all('ExampleProject') + exce = UberException.find( project: 'ExampleProject' ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end def test_api_unauth_exception - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find( project: 'ExampleProject' ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception_with_hash_in_params.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find_all('ExampleProject') + exce = UberException.find( project: 'ExampleProject' ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end def test_api_minimal_exception - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find( project: 'ExampleProject' ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/minimal_exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find_all('ExampleProject') + exce = UberException.find( project: 'ExampleProject' ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index b5c4b7f..d1d2a95 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -25,18 +25,18 @@ def test_find assert_equal uber_exception.id, UberException.get(uber_exception.id).id end - def test_find_all + def test_find exce1 = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) exce2 = UberException.occurred(create_occurrence(action_name: 'other')) Exceptionist.esclient.refresh - assert UberException.find_all('ExampleProject').include? exce1 - assert UberException.find_all('ExampleProject').include? exce2 + assert UberException.find(project: 'ExampleProject').include? exce1 + assert UberException.find(project: 'ExampleProject').include? exce2 end - def test_find_all_sorted_by_time + def test_find_sorted_by_time exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) @@ -44,11 +44,11 @@ def test_find_all_sorted_by_time Exceptionist.esclient.refresh - assert_equal [exce4, exce3, exce2, exce1], UberException.find_all_sorted_by_time('ExampleProject', 0, 10) - assert_equal [exce2], UberException.find_all_sorted_by_time('ExampleProject', 2, 1) + assert_equal [exce4, exce3, exce2, exce1], UberException.find_sorted_by_time('ExampleProject', 0, 10) + assert_equal [exce2], UberException.find_sorted_by_time('ExampleProject', 2, 1) end - def test_find_all_sorted_by_occurrences_count + def test_find_sorted_by_occurrences_count exce1 = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) @@ -59,8 +59,8 @@ def test_find_all_sorted_by_occurrences_count Exceptionist.esclient.refresh - assert_equal [exce1, exce2, exce3], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 0, 10) - assert_equal [exce2, exce3], UberException.find_all_sorted_by_occurrences_count('ExampleProject', 1, 10) + assert_equal [exce1, exce2, exce3], UberException.find_sorted_by_occurrences_count('ExampleProject', 0, 10) + assert_equal [exce2, exce3], UberException.find_sorted_by_occurrences_count('ExampleProject', 1, 10) end def test_find_new_on @@ -95,7 +95,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec, very_old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec, very_old_exec], UberException.find_sorted_by_time(project.name, 0, 20) assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # shouldn't forget anything @@ -103,7 +103,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec, very_old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec, very_old_exec], UberException.find_sorted_by_time(project.name, 0, 20) assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # should forget the very_old exception @@ -111,7 +111,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec], UberException.find_sorted_by_time(project.name, 0, 20) assert_equal [], UberException.find_new_on(project.name, very_old_date - 60) # should forget even more @@ -119,7 +119,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec], UberException.find_all_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec], UberException.find_sorted_by_time(project.name, 0, 20) assert_equal [], UberException.find_new_on(project.name, Time.now - 86400 - 3600) end @@ -140,7 +140,7 @@ def test_close! Exceptionist.esclient.refresh - assert_equal [], UberException.find_all('ExampleProject') + assert_equal [], UberException.find(project: 'ExampleProject') end def test_current_occurrence From 3f7c063165c0c6bf002f4f2f90ce5c15d5369335 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 12:52:28 +0200 Subject: [PATCH 056/293] useless assert at the beginning of the test --- test/occurrence_test.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 8585034..9761882 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -21,8 +21,6 @@ def test_delete_all_for end def test_find_first_for - assert_equal nil, Occurrence.find_first_for('empty db') - occur = create_occurrence(occurred_at: Time.local(2010, 8, 9)) create_occurrence(occurred_at: Time.local(2012, 8, 9)) other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') From 309fd4266aba2236a1e841df4c59efd8a9d95634 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 12:52:59 +0200 Subject: [PATCH 057/293] rename test --- test/uber_exception_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index d1d2a95..6e19b88 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -17,7 +17,7 @@ def test_count_all assert_equal 2, UberException.count_all('ExampleProject') end - def test_find + def test_get uber_exception = UberException.occurred(create_occurrence()) Exceptionist.esclient.refresh From 8ba9566658192f6596ac5eb82c78affbb1a50ae0 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:00:45 +0200 Subject: [PATCH 058/293] rearange class --- lib/models/occurrence.rb | 74 ++++++++++++++++++------------------ lib/models/uber_exception.rb | 5 --- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index e631466..9fa7967 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -14,43 +14,6 @@ def initialize(attributes={}) self.uber_key ||= generate_uber_key end - def inspect - "(Occurrence: id: #{id}, title: '#{title}')" - end - - def ==(other) - id == other.id - end - - def title - case exception_class - when 'Mysql::Error', 'RuntimeError', 'Timeout::Error', 'SystemExit' - "#{exception_class} #{exception_message}" - else - "#{exception_class} in #{controller_name}##{action_name}" - end - end - - def http_method - cgi_data ? cgi_data['REQUEST_METHOD'] : 'GET' - end - - def referer - cgi_data ? cgi_data['HTTP_REFERER'] : nil - end - - def user_agent - cgi_data ? cgi_data['HTTP_USER_AGENT'] : nil - end - - def occurred_at - @occurred_at.is_a?(String) ? Time.parse(@occurred_at) : @occurred_at - end - - def project - Project.new(project_name) - end - def uber_exception UberException.get(uber_key) end @@ -87,6 +50,43 @@ def self.find_all_by_name(project, size=50) Occurrence.find_all(filters: { term: { project_name: project } }, size: size) end + def ==(other) + id == other.id + end + + # + # accessors + # + + def title + case exception_class + when 'Mysql::Error', 'RuntimeError', 'Timeout::Error', 'SystemExit' + "#{exception_class} #{exception_message}" + else + "#{exception_class} in #{controller_name}##{action_name}" + end + end + + def http_method + cgi_data ? cgi_data['REQUEST_METHOD'] : 'GET' + end + + def referer + cgi_data ? cgi_data['HTTP_REFERER'] : nil + end + + def user_agent + cgi_data ? cgi_data['HTTP_USER_AGENT'] : nil + end + + def occurred_at + @occurred_at.is_a?(String) ? Time.parse(@occurred_at) : @occurred_at + end + + def project + Project.new(project_name) + end + # # serialization # diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 6182175..72d30d2 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -8,11 +8,6 @@ def initialize(attributes) @closed = attributes[:closed] end - def self.create_es(attributes) - attributes.merge!(attributes['_source']).delete('_source') - UberException.new(attributes) - end - def self.count_all(project) Exceptionist.esclient.count(type: 'exceptions', terms: { term: { project_name: project } } ) end From faca96f1cf8f0fd27c89e589ca5561d53b1e3391 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:10:04 +0200 Subject: [PATCH 059/293] add to find the size argument --- lib/models/occurrence.rb | 12 ++++++------ lib/models/uber_exception.rb | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 9fa7967..b229f7b 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -23,19 +23,19 @@ def self.delete_all_for(uber_key) end def self.find_first_for(uber_key) - Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'asc' } }) + occurrences = Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'asc' } }, size: 1) + occurrences.any? ? occurrences.first : nil end def self.find_last_for(uber_key) - Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'desc' } }) + occurrences = Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'desc' } }, size: 1) + occurrences.any? ? occurrences.first : nil end - def self.find(uber_key: '', sort: {}, position: 0) + def self.find(uber_key: '', sort: {}, position: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if position < 0 - occurrences = Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: 1 ) - - occurrences.any? ? occurrences.first : nil + Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: size ) end def self.count_all_on(project, day) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 72d30d2..4eceec3 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -80,7 +80,8 @@ def first_occurrence end def current_occurrence(position) - Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, position: position - 1) + occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, position: position - 1, size: 1) + occurrences.any? ? occurrences.first : nil end def update_occurrences_count From 483e5a8a2067e99b4a3cb0c5368dcd4f6e15a3ba Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:12:03 +0200 Subject: [PATCH 060/293] rename position to from --- lib/models/occurrence.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index b229f7b..80f5909 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -32,10 +32,10 @@ def self.find_last_for(uber_key) occurrences.any? ? occurrences.first : nil end - def self.find(uber_key: '', sort: {}, position: 0, size: 50) - raise ArgumentError, 'position has to be >= 0' if position < 0 + def self.find(uber_key: '', sort: {}, from: 0, size: 50) + raise ArgumentError, 'position has to be >= 0' if from < 0 - Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: position, size: size ) + Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: from, size: size ) end def self.count_all_on(project, day) From f08348df681b30e090b7562e0e9c73372d977a39 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:12:26 +0200 Subject: [PATCH 061/293] add argument check for variable from --- lib/models/uber_exception.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 4eceec3..1f61754 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -25,6 +25,8 @@ def self.find_sorted_by_time(project, from, size) end def self.find(project: '', filters: [], sort: {}, from: 0, size: 50) + raise ArgumentError, 'position has to be >= 0' if from < 0 + filters = [filters] if filters.class == Hash filters << { term: { closed: false } } << { term: { project_name: project } } Exceptionist.esclient.search_exceptions(filters: filters, sort: sort, from: from, size: size) From a50326e66fa021e4725b9e7842c9eae147e1587a Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:27:43 +0200 Subject: [PATCH 062/293] function to find occurrences since a date --- lib/models/occurrence.rb | 10 ++++++++-- test/occurrence_test.rb | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 80f5909..8a13e2f 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -32,10 +32,16 @@ def self.find_last_for(uber_key) occurrences.any? ? occurrences.first : nil end - def self.find(uber_key: '', sort: {}, from: 0, size: 50) + def self.find_since(uber_key, date) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, sort: { occurred_at: { order: 'desc' } }) + end + + def self.find(uber_key: '', filters: {}, sort: {}, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 - Exceptionist.esclient.search_occurrences( filters: { term: { uber_key: uber_key } }, sort: sort, from: from, size: size ) + filters = [filters] if filters.class == Hash + filters << { term: { uber_key: uber_key } } + Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) end def self.count_all_on(project, day) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 9761882..edb7fd7 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -44,6 +44,16 @@ def test_find_last_for assert_equal other_occur, Occurrence.find_last_for(other_occur.uber_key) end + def test_find_since + create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur = create_occurrence(occurred_at: Time.local(2011, 8, 9)) + new_occur = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + + Exceptionist.esclient.refresh + + assert_equal [new_occur, occur], Occurrence.find_since(occur.uber_key, Time.local(2011, 8, 9)) + end + def test_count_all_on create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) From 21ac7e6c94f2221831a677f885162d28988908ad Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:28:12 +0200 Subject: [PATCH 063/293] remove unnecessary assert --- test/occurrence_test.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index edb7fd7..ea22b63 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -32,8 +32,6 @@ def test_find_first_for end def test_find_last_for - assert_equal nil, Occurrence.find_last_for('empty db') - create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur = create_occurrence(occurred_at: Time.local(2012, 8, 9)) other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') From 632fc3a0015934a888c2e770c31463c13c56c082 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:41:51 +0200 Subject: [PATCH 064/293] remove all from method name --- lib/app.rb | 4 ++-- lib/es_helper.rb | 2 +- lib/models/occurrence.rb | 8 ++------ test/occurrence_test.rb | 16 ++++++++-------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index e47ec92..28baa9b 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -23,7 +23,7 @@ class ExceptionistApp < Sinatra::Base end get '/river' do - @occurrences = Occurrence.find_all + @occurrences = Occurrence.find @title = "River" erb :river @@ -45,7 +45,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/river' do @current_project = Project.new(params[:project]) - @occurrences = Occurrence.find_all_by_name(@current_project.name) + @occurrences = Occurrence.find_by_name(@current_project.name) @title = "Latest Occurrences for #{@current_project.name}" erb :river diff --git a/lib/es_helper.rb b/lib/es_helper.rb index d6f53b6..b4f8824 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -17,7 +17,7 @@ def self.stopCluster class Exporter def self.run - occurrences = Occurrence.find_all.map { |occurrence| occurrence.to_hash } + occurrences = Occurrence.find.map { |occurrence| occurrence.to_hash } File.open('occurrences_export.json', 'w') do |file| file.write(Yajl::Encoder.encode(occurrences)) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 8a13e2f..80f80de 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -48,12 +48,8 @@ def self.count_all_on(project, day) Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) end - def self.find_all(filters: {}, sort: { occurred_at: { order: 'desc' } }, size: 50) - Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, size: size ) - end - - def self.find_all_by_name(project, size=50) - Occurrence.find_all(filters: { term: { project_name: project } }, size: size) + def self.find_by_name(project, size=50) + Occurrence.find(filters: { term: { project_name: project } }, size: size) end def ==(other) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index ea22b63..d8ff8d8 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -10,14 +10,14 @@ def test_delete_all_for Exceptionist.esclient.refresh - assert_equal 3, Occurrence.find_all_by_name(occur.project_name).size + assert_equal 3, Occurrence.find_by_name(occur.project_name).size Occurrence.delete_all_for(occur.uber_key) Exceptionist.esclient.refresh - assert_equal 0, Occurrence.find_all_by_name(occur.project_name).size - assert_equal 1, Occurrence.find_all_by_name(other_occur.project_name).size + assert_equal 0, Occurrence.find_by_name(occur.project_name).size + assert_equal 1, Occurrence.find_by_name(other_occur.project_name).size end def test_find_first_for @@ -70,7 +70,7 @@ def test_count_all_on assert_equal 3, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) end - def test_find_all + def test_find occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) @@ -78,10 +78,10 @@ def test_find_all Exceptionist.esclient.refresh - assert_equal [occur2, occur4, occur3, occur1], Occurrence.find_all + assert_equal [occur2, occur4, occur3, occur1], Occurrence.find end - def test_find_all_by_name + def test_find_by_name occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) @@ -89,8 +89,8 @@ def test_find_all_by_name Exceptionist.esclient.refresh - assert_equal [occur2, occur3, occur1], Occurrence.find_all_by_name('ExampleProject', 5) - assert_equal [occur2, occur3], Occurrence.find_all_by_name('ExampleProject', 2) + assert_equal [occur2, occur3, occur1], Occurrence.find_by_name('ExampleProject', 5) + assert_equal [occur2, occur3], Occurrence.find_by_name('ExampleProject', 2) end def test_generate_uber_key From d4087095a320513a6101a17e1cda67a2973f995c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:42:35 +0200 Subject: [PATCH 065/293] move defautl sorting into find method --- lib/models/occurrence.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 80f80de..11b5723 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -28,15 +28,15 @@ def self.find_first_for(uber_key) end def self.find_last_for(uber_key) - occurrences = Occurrence.find(uber_key: uber_key, sort: { occurred_at: { order: 'desc' } }, size: 1) + occurrences = Occurrence.find(uber_key: uber_key, size: 1) occurrences.any? ? occurrences.first : nil end def self.find_since(uber_key, date) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, sort: { occurred_at: { order: 'desc' } }) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }) end - def self.find(uber_key: '', filters: {}, sort: {}, from: 0, size: 50) + def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash From 98404dacce57763cd397362e39a9425ee796f7fd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:43:16 +0200 Subject: [PATCH 066/293] check if empty --- lib/models/occurrence.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 11b5723..afc966f 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -40,7 +40,7 @@ def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash - filters << { term: { uber_key: uber_key } } + filters << { term: { uber_key: uber_key } } unless uber_key.empty? Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) end From 23884f29d72c72c6ab24207eb136bc527b0ffbbd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:45:39 +0200 Subject: [PATCH 067/293] reintroduce inspect method on models --- lib/models/deploy.rb | 4 ++++ lib/models/occurrence.rb | 4 ++++ lib/models/project.rb | 4 ++++ lib/models/uber_exception.rb | 3 +++ 4 files changed, 15 insertions(+) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index d1939ed..89a1425 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -38,6 +38,10 @@ def ==(other) project_name == other.project_name && version == other.version end + def inspect + "(Deploy id=#{id} project_name=#{project_name})" + end + private def to_hash { diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index afc966f..e09dfd3 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -56,6 +56,10 @@ def ==(other) id == other.id end + def inspect + "(Occurrence id=#{id} uber_key=#{uber_key})" + end + # # accessors # diff --git a/lib/models/project.rb b/lib/models/project.rb index 5dc14a3..c64805c 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -53,6 +53,10 @@ def ==(other) name == other.name end + def inspect + "(Project name=#{name})" + end + def self.find_by_key(api_key) project = Exceptionist.projects.find { |name, project_key| project_key == api_key } project ? Project.new(project.first) : nil diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 1f61754..d5d1942 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -119,4 +119,7 @@ def ==(other) id == other.id end + def inspect + "(UberException id=#{id})" + end end From 858f4ff98dbb9c3cf5d92ee88609a2f75fb9c192 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:46:04 +0200 Subject: [PATCH 068/293] fix wrong name of keyword argument --- lib/models/uber_exception.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index d5d1942..e765d0d 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -82,7 +82,7 @@ def first_occurrence end def current_occurrence(position) - occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, position: position - 1, size: 1) + occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) occurrences.any? ? occurrences.first : nil end From 1d684944fec63cfa378afb1fad5e38901f975bef Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 13:47:20 +0200 Subject: [PATCH 069/293] rearrange method --- lib/models/occurrence.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index e09dfd3..7601707 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -36,6 +36,10 @@ def self.find_since(uber_key, date) Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }) end + def self.find_by_name(project, size=50) + Occurrence.find(filters: { term: { project_name: project } }, size: size) + end + def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 @@ -48,10 +52,6 @@ def self.count_all_on(project, day) Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) end - def self.find_by_name(project, size=50) - Occurrence.find(filters: { term: { project_name: project } }, size: size) - end - def ==(other) id == other.id end From 112946483b284a2a2cedecfee5af4a8c331f4d79 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 14:14:39 +0200 Subject: [PATCH 070/293] use the word filters instead terms for keyword --- lib/es_client.rb | 6 +++--- lib/models/occurrence.rb | 13 ++++++++++++- lib/models/uber_exception.rb | 4 ++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index d4f1ba2..08b3737 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -57,9 +57,9 @@ def update(type, id, body) @es.update(index: INDEX, type: type, id: id, body: body) end - def count(type: 'occurrences', terms: []) - terms = [terms] if terms.class == Hash - query = { query: { filtered: { filter: { bool: { must: terms } } } } } + def count(type: 'occurrences', filters: []) + filters = [filters] if filters.class == Hash + query = { query: { filtered: { filter: { bool: { must: filters } } } } } @es.count(index: INDEX, type: type, body: query)['count'] end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 7601707..78cd413 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -49,7 +49,18 @@ def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } end def self.count_all_on(project, day) - Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) + Occurrence.count( project: project, filters: { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }) + end + + # def self.count_since(project, day) + # Occurrence.count_project + # Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) + # end + # + def self.count(project: '', filters: {}) + filters = [filters] if filters.class == Hash + filters << { term: { project_name: project } } unless project.empty? + Exceptionist.esclient.count( filters: filters ) end def ==(other) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index e765d0d..ce0d6f2 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -9,7 +9,7 @@ def initialize(attributes) end def self.count_all(project) - Exceptionist.esclient.count(type: 'exceptions', terms: { term: { project_name: project } } ) + Exceptionist.esclient.count(type: 'exceptions', filters: { term: { project_name: project } } ) end def self.get(uber_key) @@ -87,7 +87,7 @@ def current_occurrence(position) end def update_occurrences_count - @occurrences_count = Exceptionist.esclient.count(terms: { term: { uber_key: id } }) + @occurrences_count = Occurrence.count(filters: { term: { uber_key: id } }) Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: @occurrences_count } }) end From 60d402c24084e48ad47082d35f378bf012ea7686 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 14:25:17 +0200 Subject: [PATCH 071/293] create count since for occurrence --- lib/models/occurrence.rb | 9 ++++----- test/occurrence_test.rb | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 78cd413..6dbe7ae 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -52,11 +52,10 @@ def self.count_all_on(project, day) Occurrence.count( project: project, filters: { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }) end - # def self.count_since(project, day) - # Occurrence.count_project - # Exceptionist.esclient.count( terms: [ { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, { term: { project_name: project } } ] ) - # end - # + def self.count_since(project, time) + Occurrence.count(project: project, filters: { range: { occurred_at: { gte: time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ) + end + def self.count(project: '', filters: {}) filters = [filters] if filters.class == Hash filters << { term: { project_name: project } } unless project.empty? diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index d8ff8d8..4cf5dcc 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -70,6 +70,21 @@ def test_count_all_on assert_equal 3, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) end + def test_count_since + create_occurrence(occurred_at: Time.local(2010, 8, 9, 14, 42)) + create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) + create_occurrence(occurred_at: Time.local(2012, 8, 9, 17, 42)) + + Exceptionist.esclient.refresh + + assert_equal 2, Occurrence.count_since('ExampleProject', Time.local(2011, 8, 8)) + + create_occurrence + Exceptionist.esclient.refresh + + assert_equal 3, Occurrence.count_since('ExampleProject', Time.local(2011, 8, 8)) + end + def test_find occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) From 841cca99ab3e941873f57f3515b14849595c7b17 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 14:30:01 +0200 Subject: [PATCH 072/293] remove all from method name --- lib/models/deploy.rb | 2 +- test/api_test.rb | 4 ++-- test/deploy_test.rb | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 89a1425..79e6077 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -3,7 +3,7 @@ class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :deploy_time - def self.find_all(project) + def self.find(project) Exceptionist.esclient.search_deploys( term: { project_name: project } ) end diff --git a/test/api_test.rb b/test/api_test.rb index cb9534f..5688f9e 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -91,13 +91,13 @@ def test_api_minimal_exception end def test_api_deploy - assert_equal [], Deploy.find_all('ExampleProject') + assert_equal [], Deploy.find('ExampleProject') post '/notifier_api/v2/deploy/', read_fixtures_file('fixtures/deploy.json') assert last_response.ok? Exceptionist.esclient.refresh - assert_equal 1, Deploy.find_all('ExampleProject').count + assert_equal 1, Deploy.find('ExampleProject').count end end diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 2bf34d1..5b43c97 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -7,8 +7,8 @@ def test_save Exceptionist.esclient.refresh - assert_equal deploy, Deploy.find_all('ExampleProject').first - assert_equal 0, Deploy.find_all('OtherProject').count + assert_equal deploy, Deploy.find('ExampleProject').first + assert_equal 0, Deploy.find('OtherProject').count end def test_save_with_deploy_time @@ -16,8 +16,8 @@ def test_save_with_deploy_time Exceptionist.esclient.refresh - assert_equal deploy, Deploy.find_all('ExampleProject').first - assert_equal 0, Deploy.find_all('OtherProject').count + assert_equal deploy, Deploy.find('ExampleProject').first + assert_equal 0, Deploy.find('OtherProject').count end def test_find_all @@ -27,8 +27,8 @@ def test_find_all Exceptionist.esclient.refresh - assert_equal 2, Deploy.find_all('ExampleProject').count - assert_equal 1, Deploy.find_all('OtherProject').count + assert_equal 2, Deploy.find('ExampleProject').count + assert_equal 1, Deploy.find('OtherProject').count end def test_find_last_deploy From 851fd81e269761fefc9c7cd257d656cc05c2a120 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 15:47:03 +0200 Subject: [PATCH 073/293] create method find since last deploy --- lib/models/uber_exception.rb | 10 +++++++--- test/uber_exception_test.rb | 26 +++++++++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index ce0d6f2..93a8d4e 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -20,11 +20,15 @@ def self.find_sorted_by_occurrences_count(project, from, size) UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end - def self.find_sorted_by_time(project, from, size) - UberException.find(project: project, sort: { last_occurred_at: { order: 'desc'} }, from: from, size: size) + def self.find_since_last_deploy(project) + deploy = Deploy.find_last_deploy(project) + occurrences = Occurrence.find( filters: [ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time } } } ] ) + ids = [] + occurrences.each { |occurr| ids << occurr.uber_key } + find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) end - def self.find(project: '', filters: [], sort: {}, from: 0, size: 50) + def self.find(project: '', filters: [], sort: { last_occurred_at: { order: 'desc'} }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 6e19b88..79304ad 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -36,7 +36,7 @@ def test_find assert UberException.find(project: 'ExampleProject').include? exce2 end - def test_find_sorted_by_time + def test_find exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) @@ -44,8 +44,8 @@ def test_find_sorted_by_time Exceptionist.esclient.refresh - assert_equal [exce4, exce3, exce2, exce1], UberException.find_sorted_by_time('ExampleProject', 0, 10) - assert_equal [exce2], UberException.find_sorted_by_time('ExampleProject', 2, 1) + assert_equal [exce4, exce3, exce2, exce1], UberException.find(project: 'ExampleProject') + assert_equal [exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) end def test_find_sorted_by_occurrences_count @@ -63,6 +63,18 @@ def test_find_sorted_by_occurrences_count assert_equal [exce2, exce3], UberException.find_sorted_by_occurrences_count('ExampleProject', 1, 10) end + def test_find_since_last_deploy + exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action3')) + + create_deploy(deploy_time: Time.local(2011, 8, 13)) + + Exceptionist.esclient.refresh + + assert_equal [exce3, exce2], UberException.find_since_last_deploy('ExampleProject') + end + def test_find_new_on project = Project.new('ExampleProject') @@ -95,7 +107,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec, very_old_exec], UberException.find_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec, very_old_exec], UberException.find(project: project.name) assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # shouldn't forget anything @@ -103,7 +115,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec, very_old_exec], UberException.find_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec, very_old_exec], UberException.find(project: project.name) assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) # should forget the very_old exception @@ -111,7 +123,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec], UberException.find_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec, old_exec], UberException.find(project: project.name) assert_equal [], UberException.find_new_on(project.name, very_old_date - 60) # should forget even more @@ -119,7 +131,7 @@ def test_forget_old_exceptions Exceptionist.esclient.refresh - assert_equal [today_exec], UberException.find_sorted_by_time(project.name, 0, 20) + assert_equal [today_exec], UberException.find(project: project.name) assert_equal [], UberException.find_new_on(project.name, Time.now - 86400 - 3600) end From bb8aaf0b57291d82a119c80e25228c56b0285e91 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 15:47:37 +0200 Subject: [PATCH 074/293] fix method call --- lib/models/project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/project.rb b/lib/models/project.rb index c64805c..c5b9a88 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -33,8 +33,8 @@ def occurrence_count_on(date) Occurrence.count_all_on(name, date) end - def latest_exceptions(start, limit = 25) - UberException.find_sorted_by_time(name, start, limit) + def latest_exceptions(from, size = 25) + UberException.find(project: name, from: from, size: size) end def most_frequest_exceptions(start, limit = 25) From baa0406f039f42373a75f905605c553fe516e0ad Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 11 Sep 2014 15:58:18 +0200 Subject: [PATCH 075/293] improve tests --- test/uber_exception_test.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 79304ad..4b70ad7 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -36,7 +36,7 @@ def test_find assert UberException.find(project: 'ExampleProject').include? exce2 end - def test_find + def test_find_2 exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) @@ -64,7 +64,7 @@ def test_find_sorted_by_occurrences_count end def test_find_since_last_deploy - exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action3')) @@ -73,6 +73,14 @@ def test_find_since_last_deploy Exceptionist.esclient.refresh assert_equal [exce3, exce2], UberException.find_since_last_deploy('ExampleProject') + + UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action4')) + exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action4')) + + Exceptionist.esclient.refresh + + assert_equal [exce3, exce2, exce4], UberException.find_since_last_deploy('ExampleProject') end def test_find_new_on From a7611d83b98c823cd9b8b76621563969447f6de1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:13:43 +0200 Subject: [PATCH 076/293] add deploy data when importing --- lib/es_helper.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index b4f8824..a54bfb3 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -27,6 +27,9 @@ def self.run class Importer def self.run + `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy_old.json localhost:9292/notifier_api/v2/deploy/` + `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy.json localhost:9292/notifier_api/v2/deploy/` + files = Dir.glob('test/fixtures/occurrences_export*').sort files.each do |file| puts "importing #{file}" From d27bacd763100a62ae8dc99aef371a6a6a66800d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:14:31 +0200 Subject: [PATCH 077/293] call exception model directly --- lib/app.rb | 7 +++---- lib/models/project.rb | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 28baa9b..8c7d67d 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -34,11 +34,10 @@ class ExceptionistApp < Sinatra::Base @current_project = Project.new(params[:project]) @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' - @uber_exceptions = @current_project.most_frequest_exceptions(@start) + @uber_exceptions = UberException.find_sorted_by_occurrences_count(@current_project.name) else - @uber_exceptions = @current_project.latest_exceptions(@start) + @uber_exceptions = UberException.find(project: @current_project.name, from: @start) end - @title = "Latest Exceptions for #{@current_project.name}" erb :index end @@ -54,7 +53,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/new_on/:day' do @day = Time.parse(params[:day]) @current_project = Project.new(params[:project]) - @uber_exceptions = @current_project.new_exceptions_on(@day) + @uber_exceptions = UberException.find_new_on(@current_project.name, @day) message_body = erb(:new_exceptions, :layout => false) diff --git a/lib/models/project.rb b/lib/models/project.rb index c5b9a88..6fa053b 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -10,7 +10,7 @@ def exceptions_count end def last_thirty_days - Project.last_n_days(30).map { |day| [day, occurrence_count_on(day)] } + Project.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } end def self.last_n_days(days) From fed5f8d69596476fcd4dbe90cb84c3b8228058b3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:17:02 +0200 Subject: [PATCH 078/293] remove unused methods --- lib/models/project.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/models/project.rb b/lib/models/project.rb index 6fa053b..00744bd 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -29,22 +29,6 @@ def last_deploy Deploy.find_last_deploy(name) end - def occurrence_count_on(date) - Occurrence.count_all_on(name, date) - end - - def latest_exceptions(from, size = 25) - UberException.find(project: name, from: from, size: size) - end - - def most_frequest_exceptions(start, limit = 25) - UberException.find_sorted_by_occurrences_count(name, start, limit) - end - - def new_exceptions_on(day) - UberException.find_new_on(name, day) - end - def total_count_on(day) Occurrence.count_all_on(name, day) end From da5cca89290b4c7946ec5875513ec76a99eb6aa3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:17:37 +0200 Subject: [PATCH 079/293] add test for project --- test/project_test.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/project_test.rb b/test/project_test.rb index df5f91c..8e8f305 100644 --- a/test/project_test.rb +++ b/test/project_test.rb @@ -7,6 +7,13 @@ def test_last_n_days assert_equal 4, Project.last_n_days(4).size end + def test_exceptions_count + project = Project.new("ExampleProject") + + assert_equal 0, project.exceptions_count + end + + def test_last_thirty_days project = Project.new('ExampleProject') @@ -21,6 +28,20 @@ def test_last_thirty_days assert_equal 1, project.last_thirty_days.last[1] end + def test_last_deploy + project = Project.new("ExampleProject") + + assert_equal nil, project.last_deploy + end + + def test_find_by_key + assert_equal nil, Project.find_by_key('test_key') + + Exceptionist.add_project('project', 'test_key') + + assert_equal 'project', Project.find_by_key('test_key').name + end + def test_find_by_key assert_equal nil, Project.find_by_key('test_key') From 102574b5501cda615d5f97d95019931789edbf70 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:21:28 +0200 Subject: [PATCH 080/293] fix wrong updating of last_occurred field --- lib/models/uber_exception.rb | 15 +++++++-------- test/uber_exception_test.rb | 25 +++++++++++++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 93a8d4e..d7bbec1 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -6,6 +6,7 @@ def initialize(attributes) @project_name = attributes[:project_name] @occurrences_count = attributes[:occurrences_count] @closed = attributes[:closed] + @last_occurred_at = Time.parse(attributes[:last_occurred_at]) end def self.count_all(project) @@ -48,8 +49,10 @@ def self.find_new_on(project, day) def self.occurred(occurrence) - Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1', upsert: - { project_name: occurrence.project_name, last_occurred_at: occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), closed: false, occurrences_count: 1 } }) + timestamp = occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.last_occurred_at=last_occurred_at; ctx._source.closed=closed', + upsert: { project_name: occurrence.project_name, last_occurred_at: timestamp, closed: false, occurrences_count: 1}, + params: { last_occurred_at: timestamp, closed: false} }) Exceptionist.esclient.get_exception(occurrence.uber_key) end @@ -91,18 +94,14 @@ def current_occurrence(position) end def update_occurrences_count - @occurrences_count = Occurrence.count(filters: { term: { uber_key: id } }) - Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: @occurrences_count } }) + occurrences_count = Occurrence.count(filters: { term: { uber_key: id } }) + Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: occurrences_count } }) end def first_occurred_at first_occurrence.occurred_at end - def last_occurred_at - last_occurrence.occurred_at - end - def title last_occurrence.title end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 4b70ad7..843ae12 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -48,6 +48,20 @@ def test_find_2 assert_equal [exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) end + def test_update_last_occurred_at + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12))) + + Exceptionist.esclient.refresh + + assert_equal Time.local(2011, 8, 12), UberException.find(project: 'ExampleProject').first.last_occurred_at + + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13))) + + Exceptionist.esclient.refresh + + assert_equal Time.local(2011, 8, 13), UberException.find(project: 'ExampleProject').first.last_occurred_at + end + def test_find_sorted_by_occurrences_count exce1 = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) @@ -65,8 +79,10 @@ def test_find_sorted_by_occurrences_count def test_find_since_last_deploy UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) - exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) create_deploy(deploy_time: Time.local(2011, 8, 13)) @@ -76,11 +92,12 @@ def test_find_since_last_deploy UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action4')) UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action4')) - exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action4')) + exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action2')) Exceptionist.esclient.refresh - assert_equal [exce3, exce2, exce4], UberException.find_since_last_deploy('ExampleProject') + assert_equal [exce2, exce4, exce3], UberException.find_since_last_deploy('ExampleProject') end def test_find_new_on From 3f657dc5ffd0cb19f7a19551408b425470ee7287 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:22:08 +0200 Subject: [PATCH 081/293] add default values --- lib/models/uber_exception.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index d7bbec1..c4a6e38 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -17,7 +17,7 @@ def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end - def self.find_sorted_by_occurrences_count(project, from, size) + def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end From b20b1eee79278230621233b789c42c636a0350e2 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:22:27 +0200 Subject: [PATCH 082/293] remove duplicate testcase --- test/uber_exception_test.rb | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 843ae12..ebaffb0 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -26,17 +26,6 @@ def test_get end def test_find - exce1 = UberException.occurred(create_occurrence()) - UberException.occurred(create_occurrence()) - exce2 = UberException.occurred(create_occurrence(action_name: 'other')) - - Exceptionist.esclient.refresh - - assert UberException.find(project: 'ExampleProject').include? exce1 - assert UberException.find(project: 'ExampleProject').include? exce2 - end - - def test_find_2 exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) From e859ad23d1190c5deb827aafda5b382dd8f99874 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 11:23:13 +0200 Subject: [PATCH 083/293] fix typo --- test/uber_exception_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index ebaffb0..abe6757 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -187,7 +187,7 @@ def test_current_occurrence end end - def test_update_occurence_count + def test_update_occurrence_count uber_ex = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) From 3828a007f9334d15fd1a571a878328391c7d1369 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 12:50:32 +0200 Subject: [PATCH 084/293] update occurrences_count in since_last_deploy --- lib/models/uber_exception.rb | 15 ++++++++++++--- test/uber_exception_test.rb | 23 +++++++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index c4a6e38..611d93e 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -23,10 +23,19 @@ def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) def self.find_since_last_deploy(project) deploy = Deploy.find_last_deploy(project) - occurrences = Occurrence.find( filters: [ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time } } } ] ) + occurrences = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time } } } ],'uber_key') ids = [] - occurrences.each { |occurr| ids << occurr.uber_key } - find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) + occurrences.each { |occurr| ids << occurr['key'] } + exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) + occurrences.each do |occurr| + exces.each do |exce| + if occurr['key'] == exce.id + exce.occurrences_count = occurr['doc_count'] + break + end + end + end + exces end def self.find(project: '', filters: [], sort: { last_occurred_at: { order: 'desc'} }, from: 0, size: 50) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index abe6757..1bba7cb 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -68,25 +68,36 @@ def test_find_sorted_by_occurrences_count def test_find_since_last_deploy UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) + exec2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exec3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) create_deploy(deploy_time: Time.local(2011, 8, 13)) Exceptionist.esclient.refresh - assert_equal [exce3, exce2], UberException.find_since_last_deploy('ExampleProject') + uber_execs = UberException.find_since_last_deploy('ExampleProject') + + + assert_equal [exec3, exec2], uber_execs + assert_equal 1, uber_execs[0].occurrences_count + assert_equal 2, uber_execs[1].occurrences_count UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action4')) UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action4')) - exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) + exec4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action5')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action2')) Exceptionist.esclient.refresh - assert_equal [exce2, exce4, exce3], UberException.find_since_last_deploy('ExampleProject') + uber_execs = UberException.find_since_last_deploy('ExampleProject') + + assert_equal [exec2, exec4, exec3], uber_execs + assert_equal 3, uber_execs[0].occurrences_count + assert_equal 1, uber_execs[1].occurrences_count + assert_equal 1, uber_execs[2].occurrences_count end def test_find_new_on From 88aa9a0e00d5a357dea3911e09755ea92de4fa5c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 13:06:37 +0200 Subject: [PATCH 085/293] add since last deploy with ordering by occurrence --- lib/models/uber_exception.rb | 4 ++++ test/uber_exception_test.rb | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 611d93e..3fc69c5 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -38,6 +38,10 @@ def self.find_since_last_deploy(project) exces end + def self.find_since_last_deploy_ordered_by_occurrences_count(project) + find_since_last_deploy(project).sort{ |x, y| y.occurrences_count <=> x.occurrences_count } + end + def self.find(project: '', filters: [], sort: { last_occurred_at: { order: 'desc'} }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 1bba7cb..da4b9d1 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -100,6 +100,40 @@ def test_find_since_last_deploy assert_equal 1, uber_execs[2].occurrences_count end + def test_find_since_last_deploy_ordered_by_occurrences_count + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exec2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exec3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) + + create_deploy(deploy_time: Time.local(2011, 8, 13)) + + Exceptionist.esclient.refresh + + uber_execs = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + + + assert_equal [exec2, exec3], uber_execs + assert_equal 2, uber_execs[0].occurrences_count + assert_equal 1, uber_execs[1].occurrences_count + + UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action3')) + exec4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) + + Exceptionist.esclient.refresh + + uber_execs = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + + assert_equal [exec3, exec2, exec4], uber_execs + assert_equal 3, uber_execs[0].occurrences_count + assert_equal 2, uber_execs[1].occurrences_count + assert_equal 1, uber_execs[2].occurrences_count + end + def test_find_new_on project = Project.new('ExampleProject') From ee0b5dc1bc5163a7016ddec8d4a8a6bc4c25859b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 14:14:24 +0200 Subject: [PATCH 086/293] improve last occurred test --- test/uber_exception_test.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index da4b9d1..7408db2 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -38,17 +38,20 @@ def test_find end def test_update_last_occurred_at - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12))) + exec = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13))) Exceptionist.esclient.refresh - assert_equal Time.local(2011, 8, 12), UberException.find(project: 'ExampleProject').first.last_occurred_at + assert_equal Time.local(2011, 8, 13), UberException.get(exec.id).last_occurred_at - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17))) Exceptionist.esclient.refresh - assert_equal Time.local(2011, 8, 13), UberException.find(project: 'ExampleProject').first.last_occurred_at + assert_equal Time.local(2011, 8, 12), UberException.get(exec.id).first_occurred_at + assert_equal Time.local(2011, 8, 17), UberException.get(exec.id).last_occurred_at end def test_find_sorted_by_occurrences_count From e1572f2707ff58d8a8b1d99adb8bbb0e23d6edf8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 14:15:19 +0200 Subject: [PATCH 087/293] return nil when there is no deploy --- lib/models/uber_exception.rb | 1 + test/uber_exception_test.rb | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 3fc69c5..62be0c6 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -23,6 +23,7 @@ def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) def self.find_since_last_deploy(project) deploy = Deploy.find_last_deploy(project) + return nil unless deploy occurrences = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time } } } ],'uber_key') ids = [] occurrences.each { |occurr| ids << occurr['key'] } diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 7408db2..830c4f6 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -103,6 +103,11 @@ def test_find_since_last_deploy assert_equal 1, uber_execs[2].occurrences_count end + def test_find_since_last_deploy_with_no_deploy + UberException.occurred(create_occurrence) + assert_equal nil, UberException.find_since_last_deploy('ExampleProject') + end + def test_find_since_last_deploy_ordered_by_occurrences_count UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) From 9471c258091235887f17a096f7e295d1792e3bee Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 14:15:50 +0200 Subject: [PATCH 088/293] create since last deploy page --- lib/app.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/app.rb b/lib/app.rb index 8c7d67d..4fb75b7 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -42,6 +42,19 @@ class ExceptionistApp < Sinatra::Base erb :index end + get '/projects/:project/since_last_deploy' do + @projects = Project.all + @current_project = Project.new(params[:project]) + @start = params[:start] ? params[:start].to_i : 0 + if params[:sort_by] && params[:sort_by] == 'frequent' + @uber_exceptions = UberException.find_since_last_deploy_ordered_by_occurrences_count(@current_project.name) + else + @uber_exceptions = UberException.find_since_last_deploy(@current_project.name) + end + @title = "Exceptions since last deploy for #{@current_project.name}" + erb :index + end + get '/projects/:project/river' do @current_project = Project.new(params[:project]) @occurrences = Occurrence.find_by_name(@current_project.name) From e9ddd22b3e5134a27c82cedeae08596c08920174 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 14:16:11 +0200 Subject: [PATCH 089/293] ensure update of last occurred at are correct --- lib/models/uber_exception.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 62be0c6..4da02ee 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -64,6 +64,15 @@ def self.find_new_on(project, day) def self.occurred(occurrence) timestamp = occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + + #TODO maybe remove when arrival time is sorted + begin + exec = Exceptionist.esclient.get_exception(occurrence.uber_key) + timestamp = exec.last_occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") if occurrence.occurred_at < exec.last_occurred_at + rescue Elasticsearch::Transport::Transport::Errors::NotFound + + end + Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.last_occurred_at=last_occurred_at; ctx._source.closed=closed', upsert: { project_name: occurrence.project_name, last_occurred_at: timestamp, closed: false, occurrences_count: 1}, params: { last_occurred_at: timestamp, closed: false} }) From a5cdcbf407f7333507d1436b648168d571dd177c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:00:42 +0200 Subject: [PATCH 090/293] deploy time as time in model --- lib/models/deploy.rb | 1 + lib/models/uber_exception.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 79e6077..c604dab 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -26,6 +26,7 @@ def initialize(attributes={}) end self.deploy_time ||= Time.now + self.deploy_time = Time.parse(self.deploy_time) if self.deploy_time.is_a? String end def save diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 4da02ee..ba202a8 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -24,7 +24,7 @@ def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) def self.find_since_last_deploy(project) deploy = Deploy.find_last_deploy(project) return nil unless deploy - occurrences = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time } } } ],'uber_key') + occurrences = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] occurrences.each { |occurr| ids << occurr['key'] } exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) From 50f19f2a3176b1924b5ab1f6773ec49abdd9c1c6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:02:47 +0200 Subject: [PATCH 091/293] use key arguments as parameters --- lib/models/occurrence.rb | 4 ++-- test/occurrence_test.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 6dbe7ae..f83b25b 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -32,8 +32,8 @@ def self.find_last_for(uber_key) occurrences.any? ? occurrences.first : nil end - def self.find_since(uber_key, date) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }) + def self.find_since(uber_key: "", date: Time.local, from: 0, size: 50) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, from: from, size: size) end def self.find_by_name(project, size=50) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 4cf5dcc..830ea77 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -49,7 +49,7 @@ def test_find_since Exceptionist.esclient.refresh - assert_equal [new_occur, occur], Occurrence.find_since(occur.uber_key, Time.local(2011, 8, 9)) + assert_equal [new_occur, occur], Occurrence.find_since(uber_key: occur.uber_key, date: Time.local(2011, 8, 9)) end def test_count_all_on From 969c9415bb8ba4715282991db26dcd07e31f5ecd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:04:12 +0200 Subject: [PATCH 092/293] add function find next occurrence --- lib/models/occurrence.rb | 4 ++++ test/occurrence_test.rb | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index f83b25b..de77daa 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -40,6 +40,10 @@ def self.find_by_name(project, size=50) Occurrence.find(filters: { term: { project_name: project } }, size: size) end + def self.find_next(uber_key, date) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first + end + def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 830ea77..dd89ed4 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -52,6 +52,17 @@ def test_find_since assert_equal [new_occur, occur], Occurrence.find_since(uber_key: occur.uber_key, date: Time.local(2011, 8, 9)) end + def test_find_next + create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) + + Exceptionist.esclient.refresh + + assert_equal occur3, Occurrence.find_next(occur2.uber_key, Time.local(2011, 1, 1)) + assert_equal occur2, Occurrence.find_next(occur2.uber_key, Time.local(2012, 1, 1)) + end + def test_count_all_on create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) From db4bf1150f9acae5ab38c27b051dae2b358a2090 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:04:42 +0200 Subject: [PATCH 093/293] move test --- test/occurrence_test.rb | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index dd89ed4..7db6572 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -52,6 +52,18 @@ def test_find_since assert_equal [new_occur, occur], Occurrence.find_since(uber_key: occur.uber_key, date: Time.local(2011, 8, 9)) end + def test_find_by_name + occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) + occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) + occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) + create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + + Exceptionist.esclient.refresh + + assert_equal [occur2, occur3, occur1], Occurrence.find_by_name('ExampleProject', 5) + assert_equal [occur2, occur3], Occurrence.find_by_name('ExampleProject', 2) + end + def test_find_next create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) @@ -96,6 +108,7 @@ def test_count_since assert_equal 3, Occurrence.count_since('ExampleProject', Time.local(2011, 8, 8)) end + def test_find occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) @@ -107,18 +120,6 @@ def test_find assert_equal [occur2, occur4, occur3, occur1], Occurrence.find end - def test_find_by_name - occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) - create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') - - Exceptionist.esclient.refresh - - assert_equal [occur2, occur3, occur1], Occurrence.find_by_name('ExampleProject', 5) - assert_equal [occur2, occur3], Occurrence.find_by_name('ExampleProject', 2) - end - def test_generate_uber_key base_key = build_occurrence.uber_key assert_equal base_key, build_occurrence(url: 'lala.com').uber_key From e8a1748efe2f431a3dc271d6d667deffe0c7258e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:05:14 +0200 Subject: [PATCH 094/293] add func first occurrence since last deploy --- lib/models/uber_exception.rb | 6 ++++++ test/uber_exception_test.rb | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index ba202a8..00aa038 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -111,6 +111,12 @@ def first_occurrence @first_occurrence ||= Occurrence.find_first_for(id) end + def first_occurrence_since_last_deploy + deploy = Deploy.find_last_deploy(@project_name) + return nil unless deploy + @first_occurrence_since_last_deploy ||= Occurrence.find_next(@id, deploy.deploy_time) + end + def current_occurrence(position) occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) occurrences.any? ? occurrences.first : nil diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 830c4f6..0e4dae3 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -222,6 +222,23 @@ def test_close! assert_equal [], UberException.find(project: 'ExampleProject') end + def test_first_occurrence_since_last_deploy + exec = UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + assert_equal nil, exec.first_occurrence_since_last_deploy + + create_deploy + + occur = create_occurrence + UberException.occurred(occur) + UberException.occurred(create_occurrence) + Exceptionist.esclient.refresh + + assert_equal occur, exec.first_occurrence_since_last_deploy + end + def test_current_occurrence ocr1 = create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42)) ocr2 = create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42)) From 29f3d8789260e87db5745888e8c4dbf6d6f026eb Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 15 Sep 2014 16:09:31 +0200 Subject: [PATCH 095/293] rename variable --- lib/models/uber_exception.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 00aa038..25c8b72 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -24,11 +24,11 @@ def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) def self.find_since_last_deploy(project) deploy = Deploy.find_last_deploy(project) return nil unless deploy - occurrences = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') + agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] - occurrences.each { |occurr| ids << occurr['key'] } + agg_exces.each { |occurr| ids << occurr['key'] } exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) - occurrences.each do |occurr| + agg_exces.each do |occurr| exces.each do |exce| if occurr['key'] == exce.id exce.occurrences_count = occurr['doc_count'] From ca29fd4d60cab3b41183b5a4e3c6a3c2627e8f3e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 10:19:13 +0200 Subject: [PATCH 096/293] ensure occurred_at is parsed --- lib/models/occurrence.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index de77daa..709ad1d 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -10,7 +10,8 @@ def initialize(attributes={}) send("#{key}=", value) end - self.occurred_at ||= attributes['occurred_at'] || Time.now + self.occurred_at = Time.parse(attributes['occurred_at']) if attributes['occurred_at'] + self.occurred_at ||= Time.now self.uber_key ||= generate_uber_key end From 2f6fc22f2ea4735428f4f165b5f87875eb185e4e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 10:20:57 +0200 Subject: [PATCH 097/293] change count_since to query over exceptions --- lib/models/occurrence.rb | 4 ++-- test/occurrence_test.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 709ad1d..6491040 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -57,8 +57,8 @@ def self.count_all_on(project, day) Occurrence.count( project: project, filters: { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }) end - def self.count_since(project, time) - Occurrence.count(project: project, filters: { range: { occurred_at: { gte: time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ) + def self.count_since(uber_key, time) + Occurrence.count(filters: [{ range: { occurred_at: { gte: time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, { term: { uber_key: uber_key } }] ) end def self.count(project: '', filters: {}) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 7db6572..60a5abd 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -94,18 +94,18 @@ def test_count_all_on end def test_count_since - create_occurrence(occurred_at: Time.local(2010, 8, 9, 14, 42)) + occur = create_occurrence(occurred_at: Time.local(2010, 8, 9, 14, 42)) create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) create_occurrence(occurred_at: Time.local(2012, 8, 9, 17, 42)) Exceptionist.esclient.refresh - assert_equal 2, Occurrence.count_since('ExampleProject', Time.local(2011, 8, 8)) + assert_equal 2, Occurrence.count_since(occur.uber_key, Time.local(2011, 8, 8)) create_occurrence Exceptionist.esclient.refresh - assert_equal 3, Occurrence.count_since('ExampleProject', Time.local(2011, 8, 8)) + assert_equal 3, Occurrence.count_since(occur.uber_key, Time.local(2011, 8, 8)) end From cfcc164964384ee4a84dfdba9ece77ba53b9dd5d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 10:31:04 +0200 Subject: [PATCH 098/293] fix typos --- test/uber_exception_test.rb | 59 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 0e4dae3..7b82e98 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -73,34 +73,36 @@ def test_find_since_last_deploy UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exec2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exec3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) + exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) create_deploy(deploy_time: Time.local(2011, 8, 13)) Exceptionist.esclient.refresh - uber_execs = UberException.find_since_last_deploy('ExampleProject') + uber_exces = UberException.find_since_last_deploy('ExampleProject') + assert_equal [exce3, exce2], uber_exces + assert_equal 1, uber_exces[0].occurrences_count + assert_equal 2, uber_exces[1].occurrences_count - assert_equal [exec3, exec2], uber_execs - assert_equal 1, uber_execs[0].occurrences_count - assert_equal 2, uber_execs[1].occurrences_count - - UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action4')) - exec4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action5')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action4')) + exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 20), action_name: 'action4')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action2')) Exceptionist.esclient.refresh - uber_execs = UberException.find_since_last_deploy('ExampleProject') + uber_exces = UberException.find_since_last_deploy('ExampleProject') + + assert_equal [exce4, exce2, exce3], uber_exces + assert_equal 2, uber_exces[0].occurrences_count + assert_equal Time.local(2011, 8, 17), uber_exces[0].first_occurred_at.occurred_at + assert_equal 3, uber_exces[1].occurrences_count + assert_equal Time.local(2011, 8, 15), uber_exces[1].first_occurred_at.occurred_at + assert_equal 1, uber_exces[2].occurrences_count + assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at.occurred_at - assert_equal [exec2, exec4, exec3], uber_execs - assert_equal 3, uber_execs[0].occurrences_count - assert_equal 1, uber_execs[1].occurrences_count - assert_equal 1, uber_execs[2].occurrences_count end def test_find_since_last_deploy_with_no_deploy @@ -112,34 +114,35 @@ def test_find_since_last_deploy_ordered_by_occurrences_count UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exec2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exec3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) + exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) create_deploy(deploy_time: Time.local(2011, 8, 13)) Exceptionist.esclient.refresh - uber_execs = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') - assert_equal [exec2, exec3], uber_execs - assert_equal 2, uber_execs[0].occurrences_count - assert_equal 1, uber_execs[1].occurrences_count + assert_equal [exce2, exce3], uber_exces + assert_equal 2, uber_exces[0].occurrences_count + assert_equal 1, uber_exces[1].occurrences_count UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action3')) UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action3')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action3')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action3')) - exec4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) + exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) Exceptionist.esclient.refresh - uber_execs = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + + assert_equal [exce3, exce2, exce4], uber_exces + assert_equal 3, uber_exces[0].occurrences_count + assert_equal 2, uber_exces[1].occurrences_count + assert_equal 1, uber_exces[2].occurrences_count - assert_equal [exec3, exec2, exec4], uber_execs - assert_equal 3, uber_execs[0].occurrences_count - assert_equal 2, uber_execs[1].occurrences_count - assert_equal 1, uber_execs[2].occurrences_count end def test_find_new_on From 94d2f51b6c001069d965d8c86f3c057c31ea52c6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 10:36:41 +0200 Subject: [PATCH 099/293] denormalize last_occurrence in uber_exception --- lib/es_helper.rb | 23 +++++++++--------- lib/models/uber_exception.rb | 45 +++++++++++++++--------------------- test/uber_exception_test.rb | 35 ++++++++++++++-------------- views/_exceptions.erb | 2 +- views/_occurrence.erb | 2 +- 5 files changed, 51 insertions(+), 56 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index a54bfb3..22060cb 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -56,26 +56,27 @@ def self.run rescue Elasticsearch::Transport::Transport::Errors::NotFound end + occurr_prop = + {action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + occurred_at_day: { type: 'date' }, + occurred_at: { type: 'date' } } + Exceptionist.esclient.create_indices('exceptionist', { mappings: { _default_: { dynamic: 'false' }, - occurrences: { - properties: { - action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - occurred_at_day: { type: 'date' }, - occurred_at: { type: 'date' } - } }, + occurrences: { properties: occurr_prop }, exceptions:{ properties: { project_name: { type: 'string', index: 'not_analyzed' }, closed: { type: 'boolean' }, - last_occurred_at: { type: 'date' }, + last_occurrence: { properties: occurr_prop}, + first_occurred_at: { type: 'date' }, occurrences_count: {type: 'long'} } }, deploys: { diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 25c8b72..21f1b0b 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,12 +1,13 @@ class UberException - attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurred_at + attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at def initialize(attributes) @id = attributes[:id] @project_name = attributes[:project_name] @occurrences_count = attributes[:occurrences_count] @closed = attributes[:closed] - @last_occurred_at = Time.parse(attributes[:last_occurred_at]) + @last_occurrence = Occurrence.new(attributes[:last_occurrence]) + @first_occurred_at = Time.parse(attributes[:first_occurred_at]) end def self.count_all(project) @@ -28,13 +29,15 @@ def self.find_since_last_deploy(project) ids = [] agg_exces.each { |occurr| ids << occurr['key'] } exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) - agg_exces.each do |occurr| - exces.each do |exce| + exces.each do |exce| + agg_exces.each do |occurr| if occurr['key'] == exce.id exce.occurrences_count = occurr['doc_count'] + agg_exces.delete(occurr) break end end + exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time) end exces end @@ -43,7 +46,7 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project) find_since_last_deploy(project).sort{ |x, y| y.occurrences_count <=> x.occurrences_count } end - def self.find(project: '', filters: [], sort: { last_occurred_at: { order: 'desc'} }, from: 0, size: 50) + def self.find(project: '', filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 50) raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash @@ -60,22 +63,24 @@ def self.find_new_on(project, day) uber_exceptions.select { |uber_exp| uber_exp.first_occurred_at >= day && uber_exp.first_occurred_at < next_day } end - - def self.occurred(occurrence) - timestamp = occurrence.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + first_timestamp = occurrence.occurred_at - #TODO maybe remove when arrival time is sorted + #TODO maybe remove when events arrive sorted begin exec = Exceptionist.esclient.get_exception(occurrence.uber_key) - timestamp = exec.last_occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") if occurrence.occurred_at < exec.last_occurred_at + occurrence = exec.last_occurrence.occurred_at < first_timestamp ? occurrence : exec.last_occurrence + first_timestamp = first_timestamp < exec.first_occurred_at ? first_timestamp : exec.first_occurred_at rescue Elasticsearch::Transport::Transport::Errors::NotFound end - Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.last_occurred_at=last_occurred_at; ctx._source.closed=closed', - upsert: { project_name: occurrence.project_name, last_occurred_at: timestamp, closed: false, occurrences_count: 1}, - params: { last_occurred_at: timestamp, closed: false} }) + first_timestamp = first_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + hash = occurrence.to_hash + hash[:id] = occurrence.id + Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', + upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1}, + params: { occurrence: hash, timestamp: first_timestamp} }) Exceptionist.esclient.get_exception(occurrence.uber_key) end @@ -83,7 +88,7 @@ def self.forget_old_exceptions(project, days) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { last_occurred_at: { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) uber_exceptions.each do |exception| exception.forget! @@ -103,14 +108,6 @@ def close! Exceptionist.esclient.update('exceptions', @id, { doc: { closed: true } }) end - def last_occurrence - @last_occurrence ||= Occurrence.find_last_for(id) - end - - def first_occurrence - @first_occurrence ||= Occurrence.find_first_for(id) - end - def first_occurrence_since_last_deploy deploy = Deploy.find_last_deploy(@project_name) return nil unless deploy @@ -127,10 +124,6 @@ def update_occurrences_count Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: occurrences_count } }) end - def first_occurred_at - first_occurrence.occurred_at - end - def title last_occurrence.title end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 7b82e98..180cb35 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -2,6 +2,24 @@ class UberExceptionTest < AbstractTest + def test_occurred + occur = create_occurrence(occurred_at: Time.local(2011, 8, 12)) + exce = UberException.occurred(occur) + + Exceptionist.esclient.refresh + + assert_equal Time.local(2011, 8, 12), exce.first_occurred_at + assert_equal occur, exce.last_occurrence + + occur = create_occurrence(occurred_at: Time.local(2011, 8, 13)) + exce = UberException.occurred(occur) + + Exceptionist.esclient.refresh + + assert_equal Time.local(2011, 8, 12), exce.first_occurred_at + assert_equal occur, exce.last_occurrence + end + def test_count_all UberException.occurred(create_occurrence()) @@ -37,23 +55,6 @@ def test_find assert_equal [exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) end - def test_update_last_occurred_at - exec = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12))) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13))) - - Exceptionist.esclient.refresh - - assert_equal Time.local(2011, 8, 13), UberException.get(exec.id).last_occurred_at - - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16))) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17))) - - Exceptionist.esclient.refresh - - assert_equal Time.local(2011, 8, 12), UberException.get(exec.id).first_occurred_at - assert_equal Time.local(2011, 8, 17), UberException.get(exec.id).last_occurred_at - end - def test_find_sorted_by_occurrences_count exce1 = UberException.occurred(create_occurrence()) UberException.occurred(create_occurrence()) diff --git a/views/_exceptions.erb b/views/_exceptions.erb index 8555607..602fdcb 100644 --- a/views/_exceptions.erb +++ b/views/_exceptions.erb @@ -1,6 +1,6 @@ <% for uber_exc in exceptions %>
- Last: <%= format_time(uber_exc.last_occurred_at) %>
+ Last: <%= format_time(uber_exc.last_occurrence.occurred_at) %>
First: <%= format_time(uber_exc.first_occurred_at) %>
diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 511f380..5c9d403 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -1,6 +1,6 @@
-
<%= format_time(@uber_exception.first_occurrence.occurred_at) %>
+
<%= format_time(@uber_exception.first_occurred_at) %>
<% if @occurrence_position == 1 %> « Older From 081762d2862db61f02bce3b21839d05c2860a599 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 10:44:11 +0200 Subject: [PATCH 100/293] fix save time not whole occurrence --- lib/models/uber_exception.rb | 2 +- test/uber_exception_test.rb | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 21f1b0b..9489144 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -37,7 +37,7 @@ def self.find_since_last_deploy(project) break end end - exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time) + exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time).occurred_at end exces end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 180cb35..bead89a 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -98,12 +98,11 @@ def test_find_since_last_deploy assert_equal [exce4, exce2, exce3], uber_exces assert_equal 2, uber_exces[0].occurrences_count - assert_equal Time.local(2011, 8, 17), uber_exces[0].first_occurred_at.occurred_at + assert_equal Time.local(2011, 8, 17), uber_exces[0].first_occurred_at assert_equal 3, uber_exces[1].occurrences_count - assert_equal Time.local(2011, 8, 15), uber_exces[1].first_occurred_at.occurred_at + assert_equal Time.local(2011, 8, 15), uber_exces[1].first_occurred_at assert_equal 1, uber_exces[2].occurrences_count - assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at.occurred_at - + assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at end def test_find_since_last_deploy_with_no_deploy From 752605c35f96dc6df0c3192fb588595cb20f6a62 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 11:10:56 +0200 Subject: [PATCH 101/293] set default pagination again to 25 --- lib/es_client.rb | 8 ++++---- lib/models/occurrence.rb | 6 +++--- lib/models/uber_exception.rb | 4 ++-- test/integration_test.rb | 20 ++++++++++++++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 08b3737..daa60c9 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -12,23 +12,23 @@ def initialize(endpoint) @es = Elasticsearch::Client.new(host: endpoint) end - def search_occurrences(filters: {}, sort: {}, from: 0, size: 50) + def search_occurrences(filters: {}, sort: {}, from: 0, size: 25) hash = search(filters: filters, sort: sort, from: from, size: size) hash.hits.hits.map { |doc| create_occurrence(doc) } end - def search_exceptions(filters: {}, sort: {}, from: 0, size: 50) + def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) hash = search(type: TYPE_EXCEPTIONS, filters: filters, sort: sort, from: from, size: size) hash.hits.hits.map { |doc| create_exception(doc) } end - def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 50) + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: type, body: query) Hashie::Mash.new(response) end - def search_deploys(filters, sort={}, from=0, size=50) + def search_deploys(filters, sort={}, from=0, size=25) query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: TYPE_DEPLOYS, body: query) hash = Hashie::Mash.new(response) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 6491040..3e3df2b 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -33,11 +33,11 @@ def self.find_last_for(uber_key) occurrences.any? ? occurrences.first : nil end - def self.find_since(uber_key: "", date: Time.local, from: 0, size: 50) + def self.find_since(uber_key: "", date: Time.local, from: 0, size: 25) Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, from: from, size: size) end - def self.find_by_name(project, size=50) + def self.find_by_name(project, size=25) Occurrence.find(filters: { term: { project_name: project } }, size: size) end @@ -45,7 +45,7 @@ def self.find_next(uber_key, date) Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first end - def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 50) + def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 9489144..5fc1b6a 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -18,7 +18,7 @@ def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end - def self.find_sorted_by_occurrences_count(project, from = 0, size = 50) + def self.find_sorted_by_occurrences_count(project, from = 0, size = 25) UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end @@ -46,7 +46,7 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project) find_since_last_deploy(project).sort{ |x, y| y.occurrences_count <=> x.occurrences_count } end - def self.find(project: '', filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 50) + def self.find(project: '', filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 filters = [filters] if filters.class == Hash diff --git a/test/integration_test.rb b/test/integration_test.rb index 58b75f0..1e3ab8e 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -87,14 +87,30 @@ def test_with_one_exception assert_contain '# 2' end - def test_with_pagination + def test_project_pagination_recent 27.times do |i| UberException.occurred(create_occurrence(action_name:"action_#{i}")) end Exceptionist.esclient.refresh - visit '/projects/ExampleProject' + visit '/projects/ExampleProject?sort_by=latest' + assert_contain 'next page' + assert_not_contain 'previous page' + + click_link 'next page' + assert_not_contain 'next page' + assert_contain 'previous page' + end + + def test_project_pagination_frequent + 27.times do |i| + UberException.occurred(create_occurrence(action_name:"action_#{i}")) + end + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject?sort_by=frequent' assert_contain 'next page' assert_not_contain 'previous page' From aaa68b7e446c03f4fe8e31ef2f3144cca59fe1c8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 11:11:31 +0200 Subject: [PATCH 102/293] pass start argument --- lib/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/app.rb b/lib/app.rb index 4fb75b7..18336bb 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -34,7 +34,7 @@ class ExceptionistApp < Sinatra::Base @current_project = Project.new(params[:project]) @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' - @uber_exceptions = UberException.find_sorted_by_occurrences_count(@current_project.name) + @uber_exceptions = UberException.find_sorted_by_occurrences_count(@current_project.name, @start) else @uber_exceptions = UberException.find(project: @current_project.name, from: @start) end From 82e7a058cceef02f1b61203ef76beef441883729 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 11:16:40 +0200 Subject: [PATCH 103/293] use keyword arguments --- lib/app.rb | 2 +- lib/models/uber_exception.rb | 2 +- test/uber_exception_test.rb | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 18336bb..592d1bc 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -34,7 +34,7 @@ class ExceptionistApp < Sinatra::Base @current_project = Project.new(params[:project]) @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' - @uber_exceptions = UberException.find_sorted_by_occurrences_count(@current_project.name, @start) + @uber_exceptions = UberException.find_sorted_by_occurrences_count(project: @current_project.name, from: @start) else @uber_exceptions = UberException.find(project: @current_project.name, from: @start) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 5fc1b6a..dde1925 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -18,7 +18,7 @@ def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end - def self.find_sorted_by_occurrences_count(project, from = 0, size = 25) + def self.find_sorted_by_occurrences_count(project: '', from: 0, size: 25) UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index bead89a..418f5df 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -66,8 +66,8 @@ def test_find_sorted_by_occurrences_count Exceptionist.esclient.refresh - assert_equal [exce1, exce2, exce3], UberException.find_sorted_by_occurrences_count('ExampleProject', 0, 10) - assert_equal [exce2, exce3], UberException.find_sorted_by_occurrences_count('ExampleProject', 1, 10) + assert_equal [exce1, exce2, exce3], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject') + assert_equal [exce2, exce3], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject', from: 1) end def test_find_since_last_deploy From e56b30c47a40d2bd52a1e14506aa6f9b35e22b49 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 11:28:48 +0200 Subject: [PATCH 104/293] introduce naming convetion for function names --- test/integration_test.rb | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 1e3ab8e..cd460ea 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -16,11 +16,11 @@ def app ExceptionistApp end - def test_show_a_empty_dashboard + def test_dashboard_empty visit '/' end - def test_show_the_dashboard_with_one_project + def test_dashboard_with_one_project occurrence = create_occurrence UberException.occurred(occurrence) @@ -32,7 +32,7 @@ def test_show_the_dashboard_with_one_project click_link 'ExampleProject' end - def test_show_the_dashboard_with_no_deploy + def test_dashboard_with_no_deploy occurrence = create_occurrence UberException.occurred(occurrence) @@ -42,7 +42,7 @@ def test_show_the_dashboard_with_no_deploy assert_contain '- no deploy found' end - def test_show_the_dashboard_with_deploy + def test_dashboard_with_deploy occurrence = create_occurrence UberException.occurred(occurrence) create_deploy @@ -53,7 +53,7 @@ def test_show_the_dashboard_with_deploy assert_contain '- deploy:' end - def test_show_the_dashboard_with_two_projects + def test_dashboard_with_two_projects occur1 = create_occurrence(project_name: 'ExampleProject') UberException.occurred(occur1) @@ -67,14 +67,14 @@ def test_show_the_dashboard_with_two_projects assert_contain 'ExampleProject2' end - def test_with_no_exceptions + def test_projects_with_no_exceptions visit '/projects/ExampleProject' assert_contain 'Latest Exceptions for ExampleProject' assert_contain 'Are you kidding? There are no exceptions!' end - def test_with_one_exception + def test_projects_with_one_exception UberException.occurred(create_occurrence) UberException.occurred(create_occurrence) @@ -87,7 +87,7 @@ def test_with_one_exception assert_contain '# 2' end - def test_project_pagination_recent + def test_projects_pagination_latest 27.times do |i| UberException.occurred(create_occurrence(action_name:"action_#{i}")) end @@ -103,7 +103,7 @@ def test_project_pagination_recent assert_contain 'previous page' end - def test_project_pagination_frequent + def test_projects_pagination_frequent 27.times do |i| UberException.occurred(create_occurrence(action_name:"action_#{i}")) end @@ -119,7 +119,7 @@ def test_project_pagination_frequent assert_contain 'previous page' end - def test_be_sorted_by_most_recent + def test_projects_be_sorted_by_most_recent UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-03-01')) UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2009-02-01')) @@ -132,7 +132,7 @@ def test_be_sorted_by_most_recent assert_contain 'NameError in users#show' end - def test_show_new_exceptions + def test_projects_show_new_exceptions UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-07-01')) UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2010-08-01')) @@ -144,7 +144,7 @@ def test_show_new_exceptions assert_not_contain 'NameError in users#index' end - def test_forget_old_exceptions + def test_projects_forget_old_exceptions UberException.occurred(create_occurrence(action_name:'show', occurred_at:Time.now - (86400 * 50))) UberException.occurred(create_occurrence(action_name:'index', occurred_at:Time.now)) @@ -155,7 +155,7 @@ def test_forget_old_exceptions assert_contain 'Deleted exceptions: 1' end - def test_show_a_minimal_occurrence + def test_exceptions_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) @@ -170,7 +170,7 @@ def test_show_a_minimal_occurrence assert_contain 'User Agent' end - def test_paginate_occurrences + def test_exceptions_paginate_occurrences occur1 = create_occurrence(url: 'http://example.com/?show=one') occur2 = create_occurrence(url: 'http://example.com/?show=two') occur3 = create_occurrence(url: 'http://example.com/?show=three') @@ -198,7 +198,7 @@ def test_paginate_occurrences assert_not_contain 'GET http://example.com/?show=two' end - def test_be_able_to_close_an_exception + def test_projects_be_able_to_close_an_exception UberException.occurred(create_occurrence(action_name:'show')) UberException.occurred(create_occurrence(action_name:'index')) From bec1a474234ef8f8284cba66025aeaf6882607e6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 11:46:35 +0200 Subject: [PATCH 105/293] add since_last_deploy to frontend --- test/integration_test.rb | 32 ++++++++++++++++++++++++++++++++ views/dashboard.erb | 2 +- views/index.erb | 2 +- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index cd460ea..9a603cd 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -155,6 +155,38 @@ def test_projects_forget_old_exceptions assert_contain 'Deleted exceptions: 1' end + def test_projects_since_last_deploy_no_exce + visit '/projects/ExampleProject/since_last_deploy' + + assert_contain 'no exceptions' + end + + def test_projects_since_last_deploy_fresh_deploy + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + create_deploy + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject/since_last_deploy' + + assert_contain 'no exceptions' + end + + def test_projects_since_last_deploy_old_deploy + create_deploy + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject/since_last_deploy' + + assert_contain 'NameError in users#show' + end + def test_exceptions_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) diff --git a/views/dashboard.erb b/views/dashboard.erb index b4ce42f..d20df86 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -10,7 +10,7 @@ <% for project in @projects %>

<%= project.name %> (<%= project.exceptions_count %>) - <% if project.last_deploy %> - deploy: <%= format_time(project.last_deploy.occurred_at) %> <% else %> - no deploy found in db :(<% end %>

+ <% if project.last_deploy %> - deploy: <%= format_time(project.last_deploy.occurred_at) %> <% else %> - no deploy found in db :(<% end %>
<%= partial(:thirty_day_graph, :holder_div_id => "holder_#{project.name}", :data => project.last_thirty_days) %> diff --git a/views/index.erb b/views/index.erb index dcd8e24..c5f115a 100644 --- a/views/index.erb +++ b/views/index.erb @@ -1,6 +1,6 @@

<%= @title %>

-<% if ! @uber_exceptions.empty? %> +<% if @uber_exceptions && @uber_exceptions.any? %>
River From 77fe2c5579c65d2d3c812fc31428f309c111223c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:36:26 +0200 Subject: [PATCH 106/293] set occurrence_counit for pagination --- lib/app.rb | 4 ++++ views/index.erb | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 592d1bc..f24a8f9 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -32,6 +32,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project' do @projects = Project.all @current_project = Project.new(params[:project]) + @exceptions_count = UberException.count_all(@current_project.name) @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' @uber_exceptions = UberException.find_sorted_by_occurrences_count(project: @current_project.name, from: @start) @@ -45,6 +46,9 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/since_last_deploy' do @projects = Project.all @current_project = Project.new(params[:project]) + + @exceptions_count = UberException.count_since(project: @current_project.name, date: @deploy.deploy_time) + @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' @uber_exceptions = UberException.find_since_last_deploy_ordered_by_occurrences_count(@current_project.name) diff --git a/views/index.erb b/views/index.erb index c5f115a..e2f70f9 100644 --- a/views/index.erb +++ b/views/index.erb @@ -18,13 +18,13 @@
<% if @start - 25 >= 0 %> - « previous page + « previous page <% end %> 
<% p %>
- <% if @start + 25 <= @current_project.exceptions_count %> - next page » + <% if @start + 25 <= @exceptions_count %> + next page » <% end %>
From 57a5c3aff23899e8b9ef8945d2d5514cb1a30fb9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:38:13 +0200 Subject: [PATCH 107/293] raise error when no deploy --- lib/app.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/app.rb b/lib/app.rb index f24a8f9..36954a0 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -47,6 +47,9 @@ class ExceptionistApp < Sinatra::Base @projects = Project.all @current_project = Project.new(params[:project]) + @deploy = @current_project.last_deploy + raise ArgumentError, "There is no deploy for project #{@current_project.name}" unless @deploy + @exceptions_count = UberException.count_since(project: @current_project.name, date: @deploy.deploy_time) @start = params[:start] ? params[:start].to_i : 0 From f83694c1d1066fc23d6f8014059e3faa46e8939b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:39:40 +0200 Subject: [PATCH 108/293] return all possible aggs --- lib/es_client.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index daa60c9..9f66cf8 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -36,7 +36,8 @@ def search_deploys(filters, sort={}, from=0, size=25) end def search_aggs(filters, aggs) - query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs } } } } + # size set 0 for Integer.MAX_VALUE + query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) hash = Hashie::Mash.new(response) hash.aggregations.exceptions.buckets From f45820a35f02b5eb870acb42f7bbc9fdb15253fd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:40:09 +0200 Subject: [PATCH 109/293] create mget function in esclient --- lib/es_client.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/es_client.rb b/lib/es_client.rb index 9f66cf8..eb5b4ce 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -49,6 +49,12 @@ def search_ids(type, ids) hash.hits.hits.map { |doc| create_exception(doc) } end + def mget(ids: []) + response = @es.mget( index: INDEX, type: TYPE_EXCEPTIONS, body: { ids: ids } ) + hash = Hashie::Mash.new(response) + hash.docs.map { |doc| create_exception(doc) } + end + def index(type, body) response = @es.index(index: INDEX, type: type, body: body) Hashie::Mash.new(response) From 2df4194bd2e1b98ce5e1fdec379ec85053e3dfcf Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:45:38 +0200 Subject: [PATCH 110/293] fix pagination issue --- lib/models/uber_exception.rb | 20 ++++++++++++++++++-- test/uber_exception_test.rb | 31 ++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index dde1925..9125910 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -42,8 +42,24 @@ def self.find_since_last_deploy(project) exces end - def self.find_since_last_deploy_ordered_by_occurrences_count(project) - find_since_last_deploy(project).sort{ |x, y| y.occurrences_count <=> x.occurrences_count } + def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', from: 0, size: 25) + deploy = Deploy.find_last_deploy(project) + return nil unless deploy + agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') + ids = [] + agg_exces.each { |occurr| ids << occurr['key'] } + exces = Exceptionist.esclient.mget(ids: ids.slice!(from, size)) + exces.each do |exce| + agg_exces.each do |occurr| + if occurr['key'] == exce.id + exce.occurrences_count = occurr['doc_count'] + agg_exces.delete(occurr) + break + end + end + exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time).occurred_at + end + exces end def self.find(project: '', filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 418f5df..07e435b 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -111,37 +111,46 @@ def test_find_since_last_deploy_with_no_deploy end def test_find_since_last_deploy_ordered_by_occurrences_count - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action1')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) create_deploy(deploy_time: Time.local(2011, 8, 13)) Exceptionist.esclient.refresh - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject') - assert_equal [exce2, exce3], uber_exces + assert_equal [exce2, exce1], uber_exces assert_equal 2, uber_exces[0].occurrences_count assert_equal 1, uber_exces[1].occurrences_count - UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 12), action_name: 'action3')) + exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action3')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action3')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action3')) exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 20), action_name: 'action4')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 21), action_name: 'action4')) Exceptionist.esclient.refresh - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count('ExampleProject') + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject') - assert_equal [exce3, exce2, exce4], uber_exces - assert_equal 3, uber_exces[0].occurrences_count - assert_equal 2, uber_exces[1].occurrences_count - assert_equal 1, uber_exces[2].occurrences_count + assert_equal [exce3, exce4, exce2, exce1], uber_exces + assert_equal 4, uber_exces[0].occurrences_count + assert_equal 3, uber_exces[1].occurrences_count + assert_equal 2, uber_exces[2].occurrences_count + assert_equal 1, uber_exces[3].occurrences_count + + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject', size: 2) + assert_equal [exce3, exce4], uber_exces + + uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject', from: 2, size: 2) + assert_equal [exce2, exce1], uber_exces end From edde3a7d970f2aa11d44d3f37687c7bfb82abab5 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:47:29 +0200 Subject: [PATCH 111/293] add count since function --- lib/models/uber_exception.rb | 4 ++++ test/uber_exception_test.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 9125910..22516c1 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -14,6 +14,10 @@ def self.count_all(project) Exceptionist.esclient.count(type: 'exceptions', filters: { term: { project_name: project } } ) end + def self.count_since(project: '', date: '') + Exceptionist.esclient.count(type: 'exceptions', filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + end + def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 07e435b..6c16d60 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -35,6 +35,19 @@ def test_count_all assert_equal 2, UberException.count_all('ExampleProject') end + def test_count_since + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) + + Exceptionist.esclient.refresh + + assert_equal 4, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 12)) + assert_equal 2, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 15)) + assert_equal 0, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 20)) + end + def test_get uber_exception = UberException.occurred(create_occurrence()) From 856e8fd1734da7cbb02518a355cf8fb1dac276e9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:49:57 +0200 Subject: [PATCH 112/293] use keyword arguments as parameters --- lib/app.rb | 4 ++-- lib/models/uber_exception.rb | 4 ++-- test/uber_exception_test.rb | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 36954a0..8d3fabe 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -54,9 +54,9 @@ class ExceptionistApp < Sinatra::Base @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' - @uber_exceptions = UberException.find_since_last_deploy_ordered_by_occurrences_count(@current_project.name) + @uber_exceptions = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: @current_project.name, from: @start) else - @uber_exceptions = UberException.find_since_last_deploy(@current_project.name) + @uber_exceptions = UberException.find_since_last_deploy(project: @current_project.name, from: @start) end @title = "Exceptions since last deploy for #{@current_project.name}" erb :index diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 22516c1..68983ee 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -26,13 +26,13 @@ def self.find_sorted_by_occurrences_count(project: '', from: 0, size: 25) UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end - def self.find_since_last_deploy(project) + def self.find_since_last_deploy(project: '', from: 0, size: 25) deploy = Deploy.find_last_deploy(project) return nil unless deploy agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] agg_exces.each { |occurr| ids << occurr['key'] } - exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } } ) + exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } }, from: from, size: size ) exces.each do |exce| agg_exces.each do |occurr| if occurr['key'] == exce.id diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 6c16d60..bd94430 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -94,7 +94,7 @@ def test_find_since_last_deploy Exceptionist.esclient.refresh - uber_exces = UberException.find_since_last_deploy('ExampleProject') + uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') assert_equal [exce3, exce2], uber_exces assert_equal 1, uber_exces[0].occurrences_count @@ -107,7 +107,7 @@ def test_find_since_last_deploy Exceptionist.esclient.refresh - uber_exces = UberException.find_since_last_deploy('ExampleProject') + uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') assert_equal [exce4, exce2, exce3], uber_exces assert_equal 2, uber_exces[0].occurrences_count @@ -120,7 +120,7 @@ def test_find_since_last_deploy def test_find_since_last_deploy_with_no_deploy UberException.occurred(create_occurrence) - assert_equal nil, UberException.find_since_last_deploy('ExampleProject') + assert_equal nil, UberException.find_since_last_deploy(project: 'ExampleProject') end def test_find_since_last_deploy_ordered_by_occurrences_count From 8d42aba2b9f65c9dbc09e98f71b3ab71baeedd7f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:50:27 +0200 Subject: [PATCH 113/293] improve since last deploy test --- test/uber_exception_test.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index bd94430..38a525f 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -84,7 +84,7 @@ def test_find_sorted_by_occurrences_count end def test_find_since_last_deploy - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) + exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action1')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) @@ -96,7 +96,7 @@ def test_find_since_last_deploy uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') - assert_equal [exce3, exce2], uber_exces + assert_equal [exce3, exce2, exce1], uber_exces assert_equal 1, uber_exces[0].occurrences_count assert_equal 2, uber_exces[1].occurrences_count @@ -109,13 +109,21 @@ def test_find_since_last_deploy uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') - assert_equal [exce4, exce2, exce3], uber_exces + assert_equal [exce4, exce2, exce3, exce1], uber_exces assert_equal 2, uber_exces[0].occurrences_count assert_equal Time.local(2011, 8, 17), uber_exces[0].first_occurred_at assert_equal 3, uber_exces[1].occurrences_count assert_equal Time.local(2011, 8, 15), uber_exces[1].first_occurred_at assert_equal 1, uber_exces[2].occurrences_count assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at + assert_equal 1, uber_exces[3].occurrences_count + assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at + + uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject', size: 2) + assert_equal [exce4, exce2], uber_exces + + uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject', from: 2, size: 2) + assert_equal [exce3, exce1], uber_exces end def test_find_since_last_deploy_with_no_deploy From ad181e393cac9c3f7299ab1f1254283283cebef1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:51:35 +0200 Subject: [PATCH 114/293] improve empty project test --- test/integration_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 9a603cd..084ece5 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -71,7 +71,9 @@ def test_projects_with_no_exceptions visit '/projects/ExampleProject' assert_contain 'Latest Exceptions for ExampleProject' - assert_contain 'Are you kidding? There are no exceptions!' + assert_contain 'no exceptions' + assert_not_contain 'next page' + assert_not_contain 'previous page' end def test_projects_with_one_exception From dbf6429ffd3de7192296d883192eb2bf9f5c79d4 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:52:21 +0200 Subject: [PATCH 115/293] use rack to test thrown exce --- test/integration_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 084ece5..0027b82 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -157,10 +157,10 @@ def test_projects_forget_old_exceptions assert_contain 'Deleted exceptions: 1' end - def test_projects_since_last_deploy_no_exce - visit '/projects/ExampleProject/since_last_deploy' + def test_projects_since_last_deploy_with_no_deploy + get '/projects/ExampleProject/since_last_deploy' - assert_contain 'no exceptions' + assert !last_response.ok? end def test_projects_since_last_deploy_fresh_deploy From 30de483bf11a91f01c10d6b0deb844b944d8fe18 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:52:55 +0200 Subject: [PATCH 116/293] add more assertions --- test/integration_test.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index 0027b82..b451a12 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -173,6 +173,7 @@ def test_projects_since_last_deploy_fresh_deploy visit '/projects/ExampleProject/since_last_deploy' + assert_contain 'since last deploy' assert_contain 'no exceptions' end @@ -187,6 +188,8 @@ def test_projects_since_last_deploy_old_deploy visit '/projects/ExampleProject/since_last_deploy' assert_contain 'NameError in users#show' + assert_not_contain 'next page' + assert_not_contain 'previous page' end def test_exceptions_show_a_minimal_occurrence From 0eba8ae22043facf2560733326a424123b3f2ba4 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 15:54:25 +0200 Subject: [PATCH 117/293] add pagination test for since last deploy --- test/integration_test.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index b451a12..66af070 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -192,6 +192,26 @@ def test_projects_since_last_deploy_old_deploy assert_not_contain 'previous page' end + def test_projects_since_last_deploy_pagination + create_deploy + + 27.times do |i| + UberException.occurred(create_occurrence(action_name:"action_#{i}")) + end + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject/since_last_deploy' + assert_contain 'since last deploy' + assert_contain 'next page' + assert_not_contain 'previous page' + + click_link 'next page' + assert_contain 'since last deploy' + assert_not_contain 'next page' + assert_contain 'previous page' + end + def test_exceptions_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) From 7d5e55035cad9e500920f09e06dc7043f965e599 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:35:18 +0200 Subject: [PATCH 118/293] path not rechable anymore --- lib/app.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 8d3fabe..5f21823 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -97,11 +97,6 @@ class ExceptionistApp < Sinatra::Base @occurrence_position = @uber_exception.occurrences_count @occurrence = @uber_exception.current_occurrence(@occurrence_position) - if @occurrence.nil? - @uber_exception.update_occurrences_count - redirect request.url - end - @current_project = @occurrence.project @backlink = true From b94ef28e989bcd466122b6fd39ded4837124e7bd Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:36:26 +0200 Subject: [PATCH 119/293] dont inherit anymore, own setup function --- test/api_test.rb | 6 +++++- test/integration_test.rb | 6 +++++- test/project_test.rb | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/api_test.rb b/test/api_test.rb index 5688f9e..9bc6b79 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -2,9 +2,13 @@ require 'rack/test' -class ApiTest < AbstractTest +class ApiTest < MiniTest::Test include Rack::Test::Methods + def setup + clear_collections + end + def app ExceptionistApp end diff --git a/test/integration_test.rb b/test/integration_test.rb index 66af070..e3419b9 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -7,11 +7,15 @@ config.mode = :rack end -class IntegrationTest < AbstractTest +class IntegrationTest < MiniTest::Test include Rack::Test::Methods include Webrat::Methods include Webrat::Matchers + def setup + clear_collections + end + def app ExceptionistApp end diff --git a/test/project_test.rb b/test/project_test.rb index 8e8f305..42eedcb 100644 --- a/test/project_test.rb +++ b/test/project_test.rb @@ -1,6 +1,10 @@ require 'test_helper' -class ProjectTest < AbstractTest +class ProjectTest < MiniTest::Test + + def setup + clear_collections + end def test_last_n_days assert_equal 2, Project.last_n_days(2).size From 472697a17212c6ca92261ed4c4af14b195686c47 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:38:45 +0200 Subject: [PATCH 120/293] use id for comparison --- lib/models/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index c604dab..fe59b11 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -36,7 +36,7 @@ def save end def ==(other) - project_name == other.project_name && version == other.version + id == other.id end def inspect From d5b3f09acf65f4017b16823ce5cb99dc82984cc6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:39:41 +0200 Subject: [PATCH 121/293] add default value for days to forget exceptions --- lib/models/uber_exception.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 68983ee..563c2bb 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -104,7 +104,7 @@ def self.occurred(occurrence) Exceptionist.esclient.get_exception(occurrence.uber_key) end - def self.forget_old_exceptions(project, days) + def self.forget_old_exceptions(project, days=0) since_date = Time.now - (86400 * days) deleted = 0 From 116abd08c9e8ea05038756832901fe108693757c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:42:01 +0200 Subject: [PATCH 122/293] use setup function to create fixtures --- test/deploy_test.rb | 43 ++--- test/occurrence_test.rb | 123 +++++--------- test/uber_exception_test.rb | 318 ++++++++++-------------------------- 3 files changed, 133 insertions(+), 351 deletions(-) diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 5b43c97..1f4a730 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -1,46 +1,27 @@ require 'test_helper' -class DeployTest < AbstractTest +class DeployTest < MiniTest::Test - def test_save - deploy = create_deploy - - Exceptionist.esclient.refresh + def setup + clear_collections - assert_equal deploy, Deploy.find('ExampleProject').first - assert_equal 0, Deploy.find('OtherProject').count - end + @deploy11 = create_deploy(version: '0.0.1') + @deploy12 = create_deploy(version: '0.0.2') + @deploy13 = create_deploy(version: '0.0.3') - def test_save_with_deploy_time - deploy = create_deploy( deploy_time: '2014-09-10T14:45:42.125+0200' ) + @deploy21 = create_deploy(project_name: 'OtherProject', version: '1.0.0') Exceptionist.esclient.refresh - - assert_equal deploy, Deploy.find('ExampleProject').first - assert_equal 0, Deploy.find('OtherProject').count end - def test_find_all - create_deploy - create_deploy( version: '0.0.2' ) - create_deploy( project_name: 'OtherProject' ) - - Exceptionist.esclient.refresh - assert_equal 2, Deploy.find('ExampleProject').count - assert_equal 1, Deploy.find('OtherProject').count + def test_save + assert_equal [@deploy13, @deploy12, @deploy11], Deploy.find_by_project('ExampleProject') + assert_equal [@deploy21], Deploy.find_by_project('OtherProject') end def test_find_last_deploy - assert_nil Deploy.find_last_deploy('ExampleProject') - - create_deploy - deploy = create_deploy( version: '0.0.2' ) - deploy_other_project = create_deploy( project_name: 'OtherProject' ) - - Exceptionist.esclient.refresh - - assert_equal deploy, Deploy.find_last_deploy('ExampleProject') - assert_equal deploy_other_project, Deploy.find_last_deploy('OtherProject') + assert_equal @deploy13, Deploy.find_last_deploy('ExampleProject') + assert_equal @deploy21, Deploy.find_last_deploy('OtherProject') end end diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 60a5abd..e613ecb 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -1,123 +1,76 @@ require 'test_helper' -class OccurrenceTest < AbstractTest +class OccurrenceTest < MiniTest::Test - def test_delete_all_for - occur = create_occurrence(occurred_at: Time.local(2010, 8, 9)) - create_occurrence(occurred_at: Time.local(2012, 8, 9)) - create_occurrence(occurred_at: Time.local(2011, 8, 9)) - other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + def setup + clear_collections - Exceptionist.esclient.refresh + @occur11 = create_occurrence(occurred_at: Time.local(2011, 1, 1)) + @occur12 = create_occurrence(occurred_at: Time.local(2011, 1, 2)) + create_occurrence(occurred_at: Time.local(2011, 1, 3)) + create_occurrence(occurred_at: Time.local(2011, 1, 4)) + create_occurrence(occurred_at: Time.local(2011, 1, 5)) + create_occurrence(occurred_at: Time.local(2011, 1, 6)) + create_occurrence(occurred_at: Time.local(2011, 1, 7)) + @occur18 = create_occurrence(occurred_at: Time.local(2011, 1, 8)) + @occur19 = create_occurrence(occurred_at: Time.local(2011, 1, 9)) - assert_equal 3, Occurrence.find_by_name(occur.project_name).size + @occur21 = create_occurrence(occurred_at: Time.local(2011, 1, 4), action_name: 'save') + create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save') + create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save') + create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save') - Occurrence.delete_all_for(occur.uber_key) + @occur31 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 5), action_name: 'otherAction')) - Exceptionist.esclient.refresh + @occur41 = create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject') - assert_equal 0, Occurrence.find_by_name(occur.project_name).size - assert_equal 1, Occurrence.find_by_name(other_occur.project_name).size + Exceptionist.esclient.refresh end - def test_find_first_for - occur = create_occurrence(occurred_at: Time.local(2010, 8, 9)) - create_occurrence(occurred_at: Time.local(2012, 8, 9)) - other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') + def test_delete_all_for + Occurrence.delete_all_for(@occur11.uber_key) Exceptionist.esclient.refresh - assert_equal occur, Occurrence.find_first_for(occur.uber_key) - assert_equal other_occur, Occurrence.find_first_for(other_occur.uber_key) + assert_equal 5, Occurrence.find_by_name('ExampleProject').size + assert_equal 1, Occurrence.find_by_name('OtherProject').size end - def test_find_last_for - create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - other_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') - - Exceptionist.esclient.refresh + def test_find_first_for + assert_equal @occur11, Occurrence.find_first_for(@occur12.uber_key) + end - assert_equal occur, Occurrence.find_last_for(occur.uber_key) - assert_equal other_occur, Occurrence.find_last_for(other_occur.uber_key) + def test_find_last_for + assert_equal @occur19, Occurrence.find_last_for(@occur12.uber_key) end def test_find_since - create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur = create_occurrence(occurred_at: Time.local(2011, 8, 9)) - new_occur = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - - Exceptionist.esclient.refresh - - assert_equal [new_occur, occur], Occurrence.find_since(uber_key: occur.uber_key, date: Time.local(2011, 8, 9)) + assert_equal [@occur19, @occur18], Occurrence.find_since(uber_key: @occur18.uber_key, date: Time.local(2011, 1, 7, 12, 0)) end def test_find_by_name - occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) - create_occurrence(occurred_at: Time.local(2011, 8, 9), project_name: 'OtherProject') - - Exceptionist.esclient.refresh - - assert_equal [occur2, occur3, occur1], Occurrence.find_by_name('ExampleProject', 5) - assert_equal [occur2, occur3], Occurrence.find_by_name('ExampleProject', 2) + assert_equal 14, Occurrence.find_by_name('ExampleProject').size + assert_equal [@occur41], Occurrence.find_by_name('OtherProject') end def test_find_next - create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) - - Exceptionist.esclient.refresh - - assert_equal occur3, Occurrence.find_next(occur2.uber_key, Time.local(2011, 1, 1)) - assert_equal occur2, Occurrence.find_next(occur2.uber_key, Time.local(2012, 1, 1)) + assert_equal @occur19, Occurrence.find_next(@occur18.uber_key, Time.local(2011, 1, 8, 12, 0)) + assert_equal @occur12, Occurrence.find_next(@occur11.uber_key, Time.local(2011, 1, 1, 12, 0)) end def test_count_all_on - create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) - create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) - create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42), project_name: 'OtherProject') - - Exceptionist.esclient.refresh - - assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 10)) - assert_equal 2, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) - - create_occurrence(occurred_at: Time.local(2011, 8, 9, 9, 42)) - - Exceptionist.esclient.refresh - - assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 8)) - assert_equal 3, Occurrence.count_all_on('ExampleProject', Time.local(2011, 8, 9)) + assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 1, 20)) + assert_equal 4, Occurrence.count_all_on('ExampleProject', Time.local(2011, 1, 6)) end def test_count_since - occur = create_occurrence(occurred_at: Time.local(2010, 8, 9, 14, 42)) - create_occurrence(occurred_at: Time.local(2011, 8, 9, 17, 42)) - create_occurrence(occurred_at: Time.local(2012, 8, 9, 17, 42)) - - Exceptionist.esclient.refresh - - assert_equal 2, Occurrence.count_since(occur.uber_key, Time.local(2011, 8, 8)) - - create_occurrence - Exceptionist.esclient.refresh - - assert_equal 3, Occurrence.count_since(occur.uber_key, Time.local(2011, 8, 8)) + assert_equal 2, Occurrence.count_since(@occur11.uber_key, Time.local(2011, 1, 7, 12, 0)) + assert_equal 0, Occurrence.count_since(@occur21.uber_key, Time.local(2011, 1, 8)) end def test_find - occur1 = create_occurrence(occurred_at: Time.local(2010, 8, 9)) - occur2 = create_occurrence(occurred_at: Time.local(2012, 8, 9)) - occur3 = create_occurrence(occurred_at: Time.local(2011, 8, 9)) - occur4 = create_occurrence(occurred_at: Time.local(2011, 8, 10), project_name: 'OtherProject') - - Exceptionist.esclient.refresh - - assert_equal [occur2, occur4, occur3, occur1], Occurrence.find + assert_equal 15, Occurrence.find.size end def test_generate_uber_key diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 38a525f..f96a813 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -1,322 +1,170 @@ require 'test_helper' -class UberExceptionTest < AbstractTest +class UberExceptionTest < MiniTest::Test - def test_occurred - occur = create_occurrence(occurred_at: Time.local(2011, 8, 12)) - exce = UberException.occurred(occur) - - Exceptionist.esclient.refresh + def setup + clear_collections - assert_equal Time.local(2011, 8, 12), exce.first_occurred_at - assert_equal occur, exce.last_occurrence + @occur1 = create_occurrence(occurred_at: Time.local(2011, 1, 1)) + @occur2 = create_occurrence(occurred_at: Time.local(2011, 1, 2)) + UberException.occurred(@occur1) + UberException.occurred(@occur2) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 3))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 5))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 7))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 9))) + @exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 10))) - occur = create_occurrence(occurred_at: Time.local(2011, 8, 13)) - exce = UberException.occurred(occur) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4), action_name: 'save')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save')) + @exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save')) - Exceptionist.esclient.refresh - - assert_equal Time.local(2011, 8, 12), exce.first_occurred_at - assert_equal occur, exce.last_occurrence - end + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 2), action_name: 'delete')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4), action_name: 'delete')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'delete')) + @exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), action_name: 'delete')) - def test_count_all - UberException.occurred(create_occurrence()) + @exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 5), action_name: 'otherAction')) - Exceptionist.esclient.refresh + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4), project_name: 'OtherProject')) + @exce5 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject')) - assert_equal 1, UberException.count_all('ExampleProject') + create_deploy(deploy_time: Time.local(2011, 1, 5, 12, 0)) - UberException.occurred(create_occurrence(action_name: 'other')) - UberException.occurred(create_occurrence(project_name: 'OtherProject')) + create_deploy(deploy_time: Time.local(2011, 1, 6), project_name: 'OtherProject') Exceptionist.esclient.refresh - - assert_equal 2, UberException.count_all('ExampleProject') end - def test_count_since - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12), action_name: 'action1')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action2')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) + def test_occurred + assert_equal Time.local(2011, 1, 1), @exce1.first_occurred_at + assert_equal Time.local(2011, 1, 10), @exce1.last_occurrence.occurred_at + end - Exceptionist.esclient.refresh + def test_count_all + assert_equal 4, UberException.count_all('ExampleProject') + assert_equal 1, UberException.count_all('OtherProject') + end - assert_equal 4, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 12)) - assert_equal 2, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 15)) - assert_equal 0, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 8, 20)) + def test_count_since + assert_equal 4, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 1, 1)) + assert_equal 2, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 1, 7)) + assert_equal 1, UberException.count_since(project: 'ExampleProject', date: Time.local(2011, 1, 9)) end def test_get - uber_exception = UberException.occurred(create_occurrence()) - - Exceptionist.esclient.refresh - - assert_equal uber_exception.id, UberException.get(uber_exception.id).id + assert_equal @exce1.id, UberException.get(@exce1.id).id end def test_find - exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42), action_name: 'action1')) - exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42), action_name: 'action2')) - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42), action_name: 'action3')) - exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14, 14, 42), action_name: 'action4')) - - Exceptionist.esclient.refresh - - assert_equal [exce4, exce3, exce2, exce1], UberException.find(project: 'ExampleProject') - assert_equal [exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) + assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(project: 'ExampleProject') + assert_equal [@exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) end def test_find_sorted_by_occurrences_count - exce1 = UberException.occurred(create_occurrence()) - UberException.occurred(create_occurrence()) - UberException.occurred(create_occurrence()) - exce3 = UberException.occurred(create_occurrence(action_name: 'different')) - exce2 = UberException.occurred(create_occurrence(action_name: 'other')) - UberException.occurred(create_occurrence(action_name: 'other')) - - - Exceptionist.esclient.refresh - - assert_equal [exce1, exce2, exce3], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject') - assert_equal [exce2, exce3], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject', from: 1) + assert_equal [@exce1, @exce2, @exce3, @exce4], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject') end def test_find_since_last_deploy - exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action1')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 16), action_name: 'action3')) - - create_deploy(deploy_time: Time.local(2011, 8, 13)) - - Exceptionist.esclient.refresh uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') - assert_equal [exce3, exce2, exce1], uber_exces - assert_equal 1, uber_exces[0].occurrences_count + assert_equal [@exce1, @exce3, @exce2], uber_exces + assert_equal 5, uber_exces[0].occurrences_count + assert_equal Time.local(2011, 1, 6), uber_exces[0].first_occurred_at assert_equal 2, uber_exces[1].occurrences_count + assert_equal Time.local(2011, 1, 6), uber_exces[1].first_occurred_at + assert_equal 4, uber_exces[2].occurrences_count + assert_equal Time.local(2011, 1, 6), uber_exces[2].first_occurred_at - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action4')) - exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 20), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action2')) - - Exceptionist.esclient.refresh - - uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject') + uber_exces = UberException.find_since_last_deploy(project: 'OtherProject') - assert_equal [exce4, exce2, exce3, exce1], uber_exces - assert_equal 2, uber_exces[0].occurrences_count - assert_equal Time.local(2011, 8, 17), uber_exces[0].first_occurred_at - assert_equal 3, uber_exces[1].occurrences_count - assert_equal Time.local(2011, 8, 15), uber_exces[1].first_occurred_at - assert_equal 1, uber_exces[2].occurrences_count - assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at - assert_equal 1, uber_exces[3].occurrences_count - assert_equal Time.local(2011, 8, 16), uber_exces[2].first_occurred_at - - uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject', size: 2) - assert_equal [exce4, exce2], uber_exces - - uber_exces = UberException.find_since_last_deploy(project: 'ExampleProject', from: 2, size: 2) - assert_equal [exce3, exce1], uber_exces + assert_equal [@exce5], uber_exces + assert_equal 1, uber_exces[0].occurrences_count + assert_equal Time.local(2011, 1, 8), uber_exces[0].first_occurred_at end def test_find_since_last_deploy_with_no_deploy - UberException.occurred(create_occurrence) - assert_equal nil, UberException.find_since_last_deploy(project: 'ExampleProject') + assert_equal nil, UberException.find_since_last_deploy(project: 'NotExistingProject') end def test_find_since_last_deploy_ordered_by_occurrences_count - exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action1')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 11), action_name: 'action2')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action2')) - - create_deploy(deploy_time: Time.local(2011, 8, 13)) - - Exceptionist.esclient.refresh - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject') - assert_equal [exce2, exce1], uber_exces - assert_equal 2, uber_exces[0].occurrences_count - assert_equal 1, uber_exces[1].occurrences_count - - exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2010, 8, 11), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 14), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 15), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 17), action_name: 'action3')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action3')) - exce4 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 18), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 20), action_name: 'action4')) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 21), action_name: 'action4')) - - Exceptionist.esclient.refresh - - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject') - - assert_equal [exce3, exce4, exce2, exce1], uber_exces - assert_equal 4, uber_exces[0].occurrences_count - assert_equal 3, uber_exces[1].occurrences_count + assert_equal [@exce1, @exce2, @exce3], uber_exces + assert_equal 5, uber_exces[0].occurrences_count + assert_equal Time.local(2011, 1, 6), uber_exces[0].first_occurred_at + assert_equal 4, uber_exces[1].occurrences_count + assert_equal Time.local(2011, 1, 6), uber_exces[1].first_occurred_at assert_equal 2, uber_exces[2].occurrences_count - assert_equal 1, uber_exces[3].occurrences_count - - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject', size: 2) - assert_equal [exce3, exce4], uber_exces - - uber_exces = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: 'ExampleProject', from: 2, size: 2) - assert_equal [exce2, exce1], uber_exces - + assert_equal Time.local(2011, 1, 6), uber_exces[2].first_occurred_at end def test_find_new_on - project = Project.new('ExampleProject') - - old_occur = create_occurrence(occurred_at: Time.local(2011, 8, 9, 14, 42)) - yesterday_occur1 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 12, 14, 42)) - yesterday_occur2 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 12, 15, 42)) - today_occur1 = create_occurrence(action_name: 'create', occurred_at: Time.local(2011, 8, 13, 14, 42)) - today_occur2 = create_occurrence(action_name: 'index', occurred_at: Time.local(2011, 8, 13, 14, 42)) - - UberException.occurred(old_occur) - UberException.occurred(yesterday_occur1) - UberException.occurred(yesterday_occur2) - UberException.occurred(today_occur1) - UberException.occurred(today_occur2) - - Exceptionist.esclient.refresh - - exceptions = UberException.find_new_on(project.name, Time.local(2011, 8, 12)) + exceptions = UberException.find_new_on('ExampleProject', Time.local(2011, 1, 1)) assert_equal 1, exceptions.size - assert_equal [yesterday_occur1.uber_exception], exceptions + + assert_equal [], UberException.find_new_on('OtherProject', Time.local(2011, 1, 4, 12, 30)) end def test_forget_old_exceptions - project = Project.new('ExampleProject') - very_old_date = Time.now - (86400 * 50) - - very_old_exec = UberException.occurred(create_occurrence(occurred_at: very_old_date)) - old_exec = UberException.occurred(create_occurrence(action_name: 'index', occurred_at: Time.now - (86400 * 28))) - today_exec = UberException.occurred(create_occurrence(action_name: 'create', occurred_at: Time.now)) - - Exceptionist.esclient.refresh - - assert_equal [today_exec, old_exec, very_old_exec], UberException.find(project: project.name) - assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) - + days = (Time.now - Time.local(2011, 1, 1)) / 86400 # shouldn't forget anything - UberException.forget_old_exceptions(project.name, 51) - + UberException.forget_old_exceptions('ExampleProject', days) Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec, very_old_exec], UberException.find(project: project.name) - assert_equal [very_old_exec], UberException.find_new_on(project.name, very_old_date - 60) - - # should forget the very_old exception - UberException.forget_old_exceptions(project.name, 30) + assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(project: 'ExampleProject') + days = (Time.now - Time.local(2011, 1, 7)) / 86400 + UberException.forget_old_exceptions('ExampleProject', days) Exceptionist.esclient.refresh - assert_equal [today_exec, old_exec], UberException.find(project: project.name) - assert_equal [], UberException.find_new_on(project.name, very_old_date - 60) - - # should forget even more - UberException.forget_old_exceptions(project.name, 1) + assert_equal [@exce1, @exce3], UberException.find(project: 'ExampleProject') + UberException.forget_old_exceptions('ExampleProject') Exceptionist.esclient.refresh - assert_equal [today_exec], UberException.find(project: project.name) - assert_equal [], UberException.find_new_on(project.name, Time.now - 86400 - 3600) + assert_equal [], UberException.find(project: 'ExampleProject') end def test_forget! - uber_exce = UberException.occurred(create_occurrence()) - uber_exce.forget! - + @exce1.forget! Exceptionist.esclient.refresh assert_raises(Elasticsearch::Transport::Transport::Errors::NotFound) do - UberException.get(uber_exce.id) + UberException.get(@exce1.id) end end def test_close! - uber_exce = UberException.occurred(create_occurrence()) - uber_exce.close! + @exce1.close! Exceptionist.esclient.refresh - assert_equal [], UberException.find(project: 'ExampleProject') + assert_equal [], UberException.find(filters: {term: { id: @exce1.id } } ) end - def test_first_occurrence_since_last_deploy - exec = UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - - assert_equal nil, exec.first_occurrence_since_last_deploy - - create_deploy - - occur = create_occurrence - UberException.occurred(occur) - UberException.occurred(create_occurrence) - Exceptionist.esclient.refresh - - assert_equal occur, exec.first_occurrence_since_last_deploy - end def test_current_occurrence - ocr1 = create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42)) - ocr2 = create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42)) - uber_ex = UberException.occurred(ocr1) - UberException.occurred(ocr2) + assert_equal @occur1, @exce1.current_occurrence(1) + assert_equal @occur2, @exce1.current_occurrence(2) - Exceptionist.esclient.refresh - - assert_equal ocr1, uber_ex.current_occurrence(1) - assert_equal ocr2, uber_ex.current_occurrence(2) - - assert_equal nil, uber_ex.current_occurrence(3) + assert_equal nil, @exce1.current_occurrence(11) assert_raises ArgumentError do - uber_ex.current_occurrence(0) + @exce1.current_occurrence(0) end end - def test_update_occurrence_count - uber_ex = UberException.occurred(create_occurrence()) - UberException.occurred(create_occurrence()) - - Exceptionist.esclient.refresh - - Exceptionist.esclient.update('exceptions', uber_ex.id, { doc: { occurrences_count: 42 } }) - - Exceptionist.esclient.refresh - - assert_equal 42, Exceptionist.esclient.get_exception(uber_ex.id).occurrences_count - - uber_ex.update_occurrences_count - - Exceptionist.esclient.refresh - - assert_equal 2, Exceptionist.esclient.get_exception(uber_ex.id).occurrences_count - end def test_occurrences_count_on - uber_ex = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 14, 42))) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 12, 15, 42))) - UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 8, 13, 14, 42))) - - Exceptionist.esclient.refresh - - assert_equal 2, uber_ex.occurrences_count_on(Time.local(2011, 8, 12)) - assert_equal 1, uber_ex.occurrences_count_on(Time.local(2011, 8, 13)) + assert_equal 1, @exce1.occurrences_count_on(Time.local(2011, 1, 9)) + assert_equal 3, @exce2.occurrences_count_on(Time.local(2011, 1, 4)) end end From ab4aaeee60e9eb2028c4edd01cb2cb4838be0985 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:44:34 +0200 Subject: [PATCH 123/293] use keyword arguments as function params --- lib/es_client.rb | 2 +- lib/models/deploy.rb | 10 +++++++--- test/api_test.rb | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index eb5b4ce..ba27957 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -28,7 +28,7 @@ def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) Hashie::Mash.new(response) end - def search_deploys(filters, sort={}, from=0, size=25) + def search_deploys(filters: {}, sort: {}, from: 0, size: 25) query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: TYPE_DEPLOYS, body: query) hash = Hashie::Mash.new(response) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index fe59b11..788f7de 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -3,14 +3,18 @@ class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :deploy_time - def self.find(project) - Exceptionist.esclient.search_deploys( term: { project_name: project } ) + def self.find_by_project(project) + find( filters: { term: { project_name: project } } ) end def self.find_last_deploy(project) - Exceptionist.esclient.search_deploys( { term: { project_name: project } }, { deploy_time: { order: 'desc' } }, from: 0, size: 1 ).first + find( filters: { term: { project_name: project } }, from: 0, size: 1).first end + def self.find(filters: {}, sort: { deploy_time: { order: 'desc' } }, from: 0, size: 25) + raise ArgumentError, 'position has to be >= 0' if from < 0 + Exceptionist.esclient.search_deploys( filters: filters, sort: sort, from: from, size: size ) + end def self.from_json(json) attr = symbolize_keys(JSON.parse(json)) Deploy.new(attr) diff --git a/test/api_test.rb b/test/api_test.rb index 9bc6b79..21cf37c 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -95,13 +95,13 @@ def test_api_minimal_exception end def test_api_deploy - assert_equal [], Deploy.find('ExampleProject') + assert_equal [], Deploy.find_by_project('ExampleProject') post '/notifier_api/v2/deploy/', read_fixtures_file('fixtures/deploy.json') assert last_response.ok? Exceptionist.esclient.refresh - assert_equal 1, Deploy.find('ExampleProject').count + assert_equal 1, Deploy.find_by_project('ExampleProject').count end end From 0ae2e4a5273d418d5fe573b03d4e7d53cb0c5fd7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 16 Sep 2014 19:45:09 +0200 Subject: [PATCH 124/293] remove unused function --- lib/models/uber_exception.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 563c2bb..0e12fcd 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -128,12 +128,6 @@ def close! Exceptionist.esclient.update('exceptions', @id, { doc: { closed: true } }) end - def first_occurrence_since_last_deploy - deploy = Deploy.find_last_deploy(@project_name) - return nil unless deploy - @first_occurrence_since_last_deploy ||= Occurrence.find_next(@id, deploy.deploy_time) - end - def current_occurrence(position) occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) occurrences.any? ? occurrences.first : nil From 5694ea6a45aa3eb2c97a36b7cc67186bc9626729 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 09:35:07 +0200 Subject: [PATCH 125/293] add deploy.json for api test --- .gitignore | 1 - test/fixtures/deploy.json | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/deploy.json diff --git a/.gitignore b/.gitignore index 9fe02fd..ba5aa35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ config.rb dump.rdb webrat.log -*.json vendor/bundle .bundle diff --git a/test/fixtures/deploy.json b/test/fixtures/deploy.json new file mode 100644 index 0000000..c04ca98 --- /dev/null +++ b/test/fixtures/deploy.json @@ -0,0 +1,6 @@ +{ + "api_key":"SECRET_API_KEY", + "version":"0.1.0", + "changelog_link":"https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e", + "deploy_time":"2014-09-03T12:00:00.000+0200" +} From 4755f625b683e1beb38d5987cdcd5cbd1b25715f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 09:36:26 +0200 Subject: [PATCH 126/293] fix typo --- views/dashboard.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/dashboard.erb b/views/dashboard.erb index d20df86..6661412 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -10,7 +10,7 @@ <% for project in @projects %>

<%= project.name %> (<%= project.exceptions_count %>) - <% if project.last_deploy %> - deploy: <%= format_time(project.last_deploy.occurred_at) %> <% else %> - no deploy found in db :(<% end %>

+ <% if project.last_deploy %> - deploy: <%= format_time(project.last_deploy.occurred_at) %> <% else %> - no deploy found in db<% end %>
<%= partial(:thirty_day_graph, :holder_div_id => "holder_#{project.name}", :data => project.last_thirty_days) %> From 8b0f5c4aac7b6dab3e412fa155da50bac4d7756e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 09:42:51 +0200 Subject: [PATCH 127/293] create import dir --- .gitignore | 1 + lib/es_helper.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba5aa35..a22eb24 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ config.rb dump.rdb webrat.log +import vendor/bundle .bundle diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 22060cb..73ecf50 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -30,7 +30,7 @@ def self.run `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy_old.json localhost:9292/notifier_api/v2/deploy/` `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy.json localhost:9292/notifier_api/v2/deploy/` - files = Dir.glob('test/fixtures/occurrences_export*').sort + files = Dir.glob('import/occurrences_export*').sort files.each do |file| puts "importing #{file}" From 7600b944c2cf4370de957604de6a8ad0ab984791 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 09:43:12 +0200 Subject: [PATCH 128/293] import deploys from import dir with yaml --- lib/es_helper.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 73ecf50..f022815 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -1,5 +1,6 @@ require 'elasticsearch/extensions/test/cluster' require 'elasticsearch' +require 'yaml' module ESHelper @@ -27,8 +28,9 @@ def self.run class Importer def self.run - `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy_old.json localhost:9292/notifier_api/v2/deploy/` - `curl -X POST -H "Content-Type: application/json" -d @test/fixtures/deploy.json localhost:9292/notifier_api/v2/deploy/` + + puts "importing deploy.yaml" + YAML.load(File.read('import/deploy.yaml')).each { |key, value| Deploy.new(value).save} files = Dir.glob('import/occurrences_export*').sort files.each do |file| From 2a08a662fb8b719d3b31113d9bd0856a200e4373 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 10:13:34 +0200 Subject: [PATCH 129/293] add codecoverage gem --- .gitignore | 1 + Gemfile | 1 + Gemfile.lock | 7 +++++++ test/test_helper.rb | 2 ++ 4 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index a22eb24..256cb7c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ config.rb dump.rdb webrat.log import +coverage vendor/bundle .bundle diff --git a/Gemfile b/Gemfile index 4b9e070..bf3be05 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,7 @@ gem 'hashie' group :development do gem 'rack-test' + gem 'simplecov' gem 'webrat' gem 'shotgun' gem 'elasticsearch-extensions' diff --git a/Gemfile.lock b/Gemfile.lock index 1c90ff6..e74d586 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,6 +11,7 @@ GEM bson (1.10.2) bson_ext (1.10.2) bson (~> 1.10.2) + docile (1.1.5) elasticsearch (1.0.4) elasticsearch-api (= 1.0.4) elasticsearch-transport (= 1.0.4) @@ -45,6 +46,11 @@ GEM ruby-prof (0.15.1) shotgun (0.9) rack (>= 1.0) + simplecov (0.9.0) + docile (~> 1.1.0) + multi_json + simplecov-html (~> 0.8.0) + simplecov-html (0.8.0) sinatra (1.4.5) rack (~> 1.4) rack-protection (~> 1.4) @@ -78,6 +84,7 @@ DEPENDENCIES rack-test rake shotgun + simplecov sinatra unicorn webrat diff --git a/test/test_helper.rb b/test/test_helper.rb index 4b96040..2838ad7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,3 +1,5 @@ +require 'simplecov' +SimpleCov.start require 'es_helper' require 'app' From 316adc956ac3d8e2f01ff8a41b5ad952c15af9bc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 10:13:59 +0200 Subject: [PATCH 130/293] remove duplicate function --- test/project_test.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/project_test.rb b/test/project_test.rb index 42eedcb..afa4ffa 100644 --- a/test/project_test.rb +++ b/test/project_test.rb @@ -46,12 +46,4 @@ def test_find_by_key assert_equal 'project', Project.find_by_key('test_key').name end - def test_find_by_key - assert_equal nil, Project.find_by_key('test_key') - - Exceptionist.add_project('project', 'test_key') - - assert_equal 'project', Project.find_by_key('test_key').name - end - end From 5aae6932974c79b43bfc76701c9b6e03914696ad Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 10:23:46 +0200 Subject: [PATCH 131/293] ignore test files --- test/test_helper.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 2838ad7..65debbb 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,5 +1,8 @@ require 'simplecov' -SimpleCov.start +SimpleCov.start do + add_filter 'test' +end + require 'es_helper' require 'app' From 1ac7780a31aeffcde2260ba8ac31564d7b570a48 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 10:27:07 +0200 Subject: [PATCH 132/293] move spawn of es-cluster into test_helper --- lib/es_helper.rb | 12 ------------ test/test_helper.rb | 8 ++++++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/es_helper.rb b/lib/es_helper.rb index f022815..7807d20 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -4,18 +4,6 @@ module ESHelper - def self.startCluster - Elasticsearch::Extensions::Test::Cluster.start( - cluster_name: "my-testing-cluster", - port: Exceptionist.esclient.port, - nodes: 1, - ) - end - - def self.stopCluster - Elasticsearch::Extensions::Test::Cluster.stop( port: Exceptionist.esclient.port ) - end - class Exporter def self.run occurrences = Occurrence.find.map { |occurrence| occurrence.to_hash } diff --git a/test/test_helper.rb b/test/test_helper.rb index 65debbb..ab7fe00 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -7,14 +7,18 @@ require 'app' at_exit do - ESHelper.stopCluster + Elasticsearch::Extensions::Test::Cluster.stop( port: Exceptionist.esclient.port ) end # minitest install its own at_exit, so we need to do this after our own require 'minitest/autorun' Exceptionist.endpoint = "localhost:10000" -ESHelper.startCluster +Elasticsearch::Extensions::Test::Cluster.start( + cluster_name: "testing-cluster", + port: Exceptionist.esclient.port, + nodes: 1, +) ESHelper::ClearDB.run # Configure From 4002dfff8b90b33d5823eda09222c40ab6748b75 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 10:58:26 +0200 Subject: [PATCH 133/293] use utils class for rake tasks --- Rakefile | 14 +++---- lib/es_helper.rb | 45 +---------------------- utils.rb | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 51 deletions(-) create mode 100644 utils.rb diff --git a/Rakefile b/Rakefile index 695e1c2..cfbe8ee 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,6 @@ +$LOAD_PATH.unshift '.' $LOAD_PATH.unshift 'lib' -require 'tools' -require 'es_helper' +require 'utils' task :default => :test @@ -13,25 +13,25 @@ end desc "Remove a single exception with all occurrences completely" task :remove_exception do uber_key = ENV['KEY'] - Exceptionist::Remover.run(uber_key) + Utils::Remover.run(uber_key) end desc "Export occurrences in json" task :export do - ESHelper::Exporter.run + Utils::Exporter.run end desc "Import occurrences from json file" task :import do - ESHelper::Importer.run + Utils::Importer.run end desc "Clear DB and create index with mapping" task :cleardb do - ESHelper::ClearDB.run + Utils::ClearDB.run end desc "Print mapping" task :mapping do - ESHelper::Mapping.run + Utils::Mapping.run end diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 7807d20..57f9da3 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -4,41 +4,6 @@ module ESHelper - class Exporter - def self.run - occurrences = Occurrence.find.map { |occurrence| occurrence.to_hash } - - File.open('occurrences_export.json', 'w') do |file| - file.write(Yajl::Encoder.encode(occurrences)) - end - end - end - - class Importer - def self.run - - puts "importing deploy.yaml" - YAML.load(File.read('import/deploy.yaml')).each { |key, value| Deploy.new(value).save} - - files = Dir.glob('import/occurrences_export*').sort - files.each do |file| - puts "importing #{file}" - - occurrences = Yajl::Parser.parse(File.read(file)) - occurrences.each do |occurrence_hash| - - occurrence_hash.delete('uber_key') - occurrence_hash.delete('id') - - occurrence = Occurrence.new(occurrence_hash) - occurrence.save - - UberException.occurred(occurrence) - end - end - end - end - class ClearDB def self.run begin @@ -80,13 +45,5 @@ def self.run Exceptionist.esclient.refresh end end - - - class Mapping - def self.run - pp Exceptionist.esclient.get_mapping('occurrences') - puts - pp Exceptionist.esclient.get_mapping('exceptions') - end - end + end diff --git a/utils.rb b/utils.rb new file mode 100644 index 0000000..f3354fe --- /dev/null +++ b/utils.rb @@ -0,0 +1,96 @@ +require 'elasticsearch/extensions/test/cluster' +require 'elasticsearch' +require 'yaml' +require 'boot' + + +module Utils + + class Exporter + def self.run + occurrences = Occurrence.find.map { |occurrence| occurrence.to_hash } + + File.open('occurrences_export.json', 'w') do |file| + file.write(Yajl::Encoder.encode(occurrences)) + end + end + end + + class Importer + def self.run + Exceptionist.endpoint = 'localhost:9200' + + puts "importing deploy.yaml" + YAML.load(File.read('import/deploy.yaml')).each { |key, value| Deploy.new(value).save} + + files = Dir.glob('import/occurrences_export*').sort + files.each do |file| + puts "importing #{file}" + + occurrences = Yajl::Parser.parse(File.read(file)) + occurrences.each do |occurrence_hash| + + occurrence_hash.delete('uber_key') + occurrence_hash.delete('id') + + occurrence = Occurrence.new(occurrence_hash) + occurrence.save + + UberException.occurred(occurrence) + end + end + end + end + + class ClearDB + def self.run + Exceptionist.endpoint = 'localhost:9200' + begin + Exceptionist.esclient.delete_indices('exceptionist') + rescue Elasticsearch::Transport::Transport::Errors::NotFound + end + + occurr_prop = + {action_name: { type: 'string', index: 'not_analyzed' }, + controller_name: { type: 'string', index: 'not_analyzed' }, + project_name: { type: 'string', index: 'not_analyzed' }, + uber_key: { type: 'string', index: 'not_analyzed' }, + exception_class: { type: 'string', index: 'not_analyzed' }, + occurred_at_day: { type: 'date' }, + occurred_at: { type: 'date' } } + + Exceptionist.esclient.create_indices('exceptionist', + { mappings: { + _default_: { + dynamic: 'false' + }, + occurrences: { properties: occurr_prop }, + exceptions:{ + properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + closed: { type: 'boolean' }, + last_occurrence: { properties: occurr_prop}, + first_occurred_at: { type: 'date' }, + occurrences_count: {type: 'long'} + } }, + deploys: { + properties: { + project_name: { type: 'string', index: 'not_analyzed' }, + version: { type: 'string', index: 'not_analyzed' }, + deploy_time: { type: 'date' }, + changelog_link: { type: 'string', index: 'not_analyzed'} + } }, + } }) + Exceptionist.esclient.refresh + end + end + + + class Mapping + def self.run + pp Exceptionist.esclient.get_mapping('occurrences') + puts + pp Exceptionist.esclient.get_mapping('exceptions') + end + end +end From 363cd44bf2089b3851a30f0b532bc2bdefa31350 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 11:32:14 +0200 Subject: [PATCH 134/293] require yaml in boot.rb --- lib/boot.rb | 1 + lib/es_helper.rb | 1 - utils.rb | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/boot.rb b/lib/boot.rb index c922ce5..abf5e56 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -6,6 +6,7 @@ require 'mongo' require 'yajl' require 'nokogiri' +require 'yaml' require 'models/project' require 'models/uber_exception' diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 57f9da3..17394cd 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -1,6 +1,5 @@ require 'elasticsearch/extensions/test/cluster' require 'elasticsearch' -require 'yaml' module ESHelper diff --git a/utils.rb b/utils.rb index f3354fe..8b0fdb2 100644 --- a/utils.rb +++ b/utils.rb @@ -1,6 +1,5 @@ require 'elasticsearch/extensions/test/cluster' require 'elasticsearch' -require 'yaml' require 'boot' From e1ad77ae195118317e7ad6d362a693b08eb0f14c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 11:33:17 +0200 Subject: [PATCH 135/293] move mapping into a yaml file --- lib/es_helper.rb | 33 +---------------------- lib/mapping.yaml | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ utils.rb | 34 +----------------------- 3 files changed, 71 insertions(+), 65 deletions(-) create mode 100644 lib/mapping.yaml diff --git a/lib/es_helper.rb b/lib/es_helper.rb index 17394cd..1f2c877 100644 --- a/lib/es_helper.rb +++ b/lib/es_helper.rb @@ -10,39 +10,8 @@ def self.run rescue Elasticsearch::Transport::Transport::Errors::NotFound end - occurr_prop = - {action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - occurred_at_day: { type: 'date' }, - occurred_at: { type: 'date' } } - - Exceptionist.esclient.create_indices('exceptionist', - { mappings: { - _default_: { - dynamic: 'false' - }, - occurrences: { properties: occurr_prop }, - exceptions:{ - properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - closed: { type: 'boolean' }, - last_occurrence: { properties: occurr_prop}, - first_occurred_at: { type: 'date' }, - occurrences_count: {type: 'long'} - } }, - deploys: { - properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - version: { type: 'string', index: 'not_analyzed' }, - deploy_time: { type: 'date' }, - changelog_link: { type: 'string', index: 'not_analyzed'} - } }, - } }) + Exceptionist.esclient.create_indices('exceptionist',YAML.load(File.read('lib/mapping.yaml'))) Exceptionist.esclient.refresh end end - end diff --git a/lib/mapping.yaml b/lib/mapping.yaml new file mode 100644 index 0000000..7b69f62 --- /dev/null +++ b/lib/mapping.yaml @@ -0,0 +1,69 @@ +mappings: + _default_: + dynamic: 'false' + occurrences: + properties: + action_name: + type: 'string' + index: 'not_analyzed' + controller_name: + type: 'string' + index: 'not_analyzed' + project_name: + type: 'string' + index: 'not_analyzed' + uber_key: + type: 'string' + index: 'not_analyzed' + exception_class: + type: 'string' + index: 'not_analyzed' + occurred_at_day: + type: 'date' + occurred_at: + type: 'date' + exceptions: + properties: + project_name: + type: 'string' + index: 'not_analyzed' + closed: + type: 'boolean' + last_occurrence: + properties: + action_name: + type: 'string' + index: 'not_analyzed' + controller_name: + type: 'string' + index: 'not_analyzed' + project_name: + type: 'string' + index: 'not_analyzed' + uber_key: + type: 'string' + index: 'not_analyzed' + exception_class: + type: 'string' + index: 'not_analyzed' + occurred_at_day: + type: 'date' + occurred_at: + type: 'date' + first_occurred_at: + type: 'date' + occurrences_count: + type: 'long' + deploys: + properties: + project_name: + type: 'string' + index: 'not_analyzed' + version: + type: 'string' + index: 'not_analyzed' + deploy_time: + type: 'date' + changelog_link: + type: 'string' + index: 'not_analyzed' diff --git a/utils.rb b/utils.rb index 8b0fdb2..1d15f2e 100644 --- a/utils.rb +++ b/utils.rb @@ -43,48 +43,16 @@ def self.run class ClearDB def self.run - Exceptionist.endpoint = 'localhost:9200' begin Exceptionist.esclient.delete_indices('exceptionist') rescue Elasticsearch::Transport::Transport::Errors::NotFound end - occurr_prop = - {action_name: { type: 'string', index: 'not_analyzed' }, - controller_name: { type: 'string', index: 'not_analyzed' }, - project_name: { type: 'string', index: 'not_analyzed' }, - uber_key: { type: 'string', index: 'not_analyzed' }, - exception_class: { type: 'string', index: 'not_analyzed' }, - occurred_at_day: { type: 'date' }, - occurred_at: { type: 'date' } } - - Exceptionist.esclient.create_indices('exceptionist', - { mappings: { - _default_: { - dynamic: 'false' - }, - occurrences: { properties: occurr_prop }, - exceptions:{ - properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - closed: { type: 'boolean' }, - last_occurrence: { properties: occurr_prop}, - first_occurred_at: { type: 'date' }, - occurrences_count: {type: 'long'} - } }, - deploys: { - properties: { - project_name: { type: 'string', index: 'not_analyzed' }, - version: { type: 'string', index: 'not_analyzed' }, - deploy_time: { type: 'date' }, - changelog_link: { type: 'string', index: 'not_analyzed'} - } }, - } }) + Exceptionist.esclient.create_indices('exceptionist',YAML.load(File.read('lib/mapping.yaml'))) Exceptionist.esclient.refresh end end - class Mapping def self.run pp Exceptionist.esclient.get_mapping('occurrences') From e5aac672ed2e70cc7fe1b2a8b65f9065b1a031a0 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 11:41:18 +0200 Subject: [PATCH 136/293] move require elasticsearch into boot.rb --- lib/boot.rb | 1 + lib/es_client.rb | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/boot.rb b/lib/boot.rb index abf5e56..b0f0a4b 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -4,6 +4,7 @@ require 'active_support/ordered_hash' require 'mongo' +require 'elasticsearch' require 'yajl' require 'nokogiri' require 'yaml' diff --git a/lib/es_client.rb b/lib/es_client.rb index ba27957..6a9e1af 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -1,5 +1,3 @@ -require 'elasticsearch' - class ESClient attr_accessor :es, :host, :port INDEX = 'exceptionist' From 44ae9566beacfcbb60c71f5a7b35dee7096815ee Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 11:42:47 +0200 Subject: [PATCH 137/293] remove es_helper --- lib/es_helper.rb | 17 ----------------- test/test_helper.rb | 7 +++++-- 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 lib/es_helper.rb diff --git a/lib/es_helper.rb b/lib/es_helper.rb deleted file mode 100644 index 1f2c877..0000000 --- a/lib/es_helper.rb +++ /dev/null @@ -1,17 +0,0 @@ -require 'elasticsearch/extensions/test/cluster' -require 'elasticsearch' - -module ESHelper - - class ClearDB - def self.run - begin - Exceptionist.esclient.delete_indices('exceptionist') - rescue Elasticsearch::Transport::Transport::Errors::NotFound - end - - Exceptionist.esclient.create_indices('exceptionist',YAML.load(File.read('lib/mapping.yaml'))) - Exceptionist.esclient.refresh - end - end -end diff --git a/test/test_helper.rb b/test/test_helper.rb index ab7fe00..e772f4c 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -3,8 +3,9 @@ add_filter 'test' end -require 'es_helper' require 'app' +require 'elasticsearch/extensions/test/cluster' + at_exit do Elasticsearch::Extensions::Test::Cluster.stop( port: Exceptionist.esclient.port ) @@ -19,7 +20,9 @@ port: Exceptionist.esclient.port, nodes: 1, ) -ESHelper::ClearDB.run + +Exceptionist.esclient.create_indices('exceptionist', YAML.load(File.read('lib/mapping.yaml') )) +Exceptionist.esclient.refresh # Configure Exceptionist.add_project 'ExampleProject', 'SECRET_API_KEY' From b27967d35cc80f7270ca5ce738a3cecc8a24bfa9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 11:43:10 +0200 Subject: [PATCH 138/293] set es-endpoint --- utils.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.rb b/utils.rb index 1d15f2e..d2947bc 100644 --- a/utils.rb +++ b/utils.rb @@ -43,6 +43,8 @@ def self.run class ClearDB def self.run + Exceptionist.endpoint = 'localhost:9200' + begin Exceptionist.esclient.delete_indices('exceptionist') rescue Elasticsearch::Transport::Transport::Errors::NotFound From 5d86ee810cc7b75b2801cd8e63333fb881ea0458 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:20:37 +0200 Subject: [PATCH 139/293] add test for unauth deploy --- test/api_test.rb | 11 +++++++++++ test/fixtures/unauth_deploy.json | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 test/fixtures/unauth_deploy.json diff --git a/test/api_test.rb b/test/api_test.rb index 21cf37c..b880f93 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -104,4 +104,15 @@ def test_api_deploy assert_equal 1, Deploy.find_by_project('ExampleProject').count end + + def test_api_unauth_deploy + post '/notifier_api/v2/deploy/', read_fixtures_file('fixtures/unauth_deploy.json') + + assert_equal 'Invalid API Key', last_response.body + assert_equal 401, last_response.status + + Exceptionist.esclient.refresh + + assert_equal [], Deploy.find + end end diff --git a/test/fixtures/unauth_deploy.json b/test/fixtures/unauth_deploy.json new file mode 100644 index 0000000..0616c97 --- /dev/null +++ b/test/fixtures/unauth_deploy.json @@ -0,0 +1,6 @@ +{ + "api_key":"UNAUTH_KEY", + "version":"0.1.0", + "changelog_link":"https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e", + "deploy_time":"2014-09-03T12:00:00.000+0200" +} From 2e9016f8f5a237681f11d4d84580c1694a59374b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:21:10 +0200 Subject: [PATCH 140/293] fix naming of test functions --- test/api_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/api_test.rb b/test/api_test.rb index b880f93..b565e9f 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -44,7 +44,7 @@ def test_add_occurrences_if_it_is_the_same_exception assert_equal 2, exce.first.occurrences_count end - def test_check_if_api_key_is_valid + def test_api_unauth_exception post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/unauth_exception.xml') assert_equal 'Invalid API Key', last_response.body @@ -68,7 +68,7 @@ def test_api_full_exception assert_equal 1, exce.first.occurrences_count end - def test_api_unauth_exception + def test_api_exception_with_hash assert_equal [], UberException.find( project: 'ExampleProject' ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception_with_hash_in_params.xml') From 487acc7998f5fff3babfc2e42d56b84883a165d7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:21:54 +0200 Subject: [PATCH 141/293] remove unused functions --- lib/models/occurrence.rb | 11 ----------- lib/models/uber_exception.rb | 5 ----- 2 files changed, 16 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 3e3df2b..95a8db7 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -118,17 +118,6 @@ def save self end - def self.create(attributes = {}) - new(attributes).save - end - - def self.es_create(attributes) - return nil unless attributes - - attributes.merge!(attributes['_source']).delete('_source') - Occurrence.new(attributes) - end - def to_hash { exception_message: exception_message, session: session, diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 0e12fcd..761fe7d 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -133,11 +133,6 @@ def current_occurrence(position) occurrences.any? ? occurrences.first : nil end - def update_occurrences_count - occurrences_count = Occurrence.count(filters: { term: { uber_key: id } }) - Exceptionist.esclient.update('exceptions', id, { doc: { occurrences_count: occurrences_count } }) - end - def title last_occurrence.title end From 68fa3b353d9b62bd38b2bc5bdfe0d2273b3244bc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:23:59 +0200 Subject: [PATCH 142/293] add tests for river --- test/integration_test.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index e3419b9..26877b5 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -71,6 +71,26 @@ def test_dashboard_with_two_projects assert_contain 'ExampleProject2' end + def test_river + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/river' + assert_contain 'River' + end + + def test_river_project + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit 'projects/ExampleProject/river' + assert_contain 'Latest Occurrences' + end + def test_projects_with_no_exceptions visit '/projects/ExampleProject' From 4c211e02cfa505801c0403e3fb1c0823ced4108a Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:24:22 +0200 Subject: [PATCH 143/293] add tests for since last deploy --- test/integration_test.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index 26877b5..bfdd54c 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -216,6 +216,21 @@ def test_projects_since_last_deploy_old_deploy assert_not_contain 'previous page' end + def test_projects_since_last_deploy_ordered_by_occurrences_count + create_deploy + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject/since_last_deploy?sort_by=frequent' + + assert_contain 'NameError in users#show' + assert_not_contain 'next page' + assert_not_contain 'previous page' + end + def test_projects_since_last_deploy_pagination create_deploy From d67e6d06bdbdebe75586f6d1fdd7685c083e13c1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:25:18 +0200 Subject: [PATCH 144/293] add test for title --- test/occurrence_test.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index e613ecb..ec5b872 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -22,11 +22,15 @@ def setup @occur31 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 5), action_name: 'otherAction')) - @occur41 = create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject') + @occur41 = create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject', exception_class: 'Mysql::Error', exception_message: 'Line 42') Exceptionist.esclient.refresh end + def test_title_accessor + assert_equal 'Mysql::Error Line 42', @occur41.title + end + def test_delete_all_for Occurrence.delete_all_for(@occur11.uber_key) From c3ffbaefae047afb3d41e80818d484df9ad047ff Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 13:25:59 +0200 Subject: [PATCH 145/293] add test for get exception --- test/occurrence_test.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index ec5b872..41e844d 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -27,6 +27,13 @@ def setup Exceptionist.esclient.refresh end + def test_get_uber_exce + exce = UberException.occurred(@occur11) + Exceptionist.esclient.refresh + + assert_equal exce, @occur11.uber_exception + end + def test_title_accessor assert_equal 'Mysql::Error Line 42', @occur41.title end From 11913dbb9c1c25009e915d39144d52924b31f1bc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 14:28:25 +0200 Subject: [PATCH 146/293] split mapping to reuse type --- lib/boot.rb | 2 + lib/mapping.yaml | 69 ----------------------------------- lib/mapping/deploys.yaml | 12 ++++++ lib/mapping/exceptions.yaml | 10 +++++ lib/mapping/mapping_helper.rb | 19 ++++++++++ lib/mapping/occurrences.yaml | 20 ++++++++++ test/test_helper.rb | 2 +- utils.rb | 2 +- 8 files changed, 65 insertions(+), 71 deletions(-) delete mode 100644 lib/mapping.yaml create mode 100644 lib/mapping/deploys.yaml create mode 100644 lib/mapping/exceptions.yaml create mode 100644 lib/mapping/mapping_helper.rb create mode 100644 lib/mapping/occurrences.yaml diff --git a/lib/boot.rb b/lib/boot.rb index b0f0a4b..342d13d 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -15,4 +15,6 @@ require 'models/deploy' require 'models/mailer' +require 'mapping/mapping_helper' + require 'exceptionist' diff --git a/lib/mapping.yaml b/lib/mapping.yaml deleted file mode 100644 index 7b69f62..0000000 --- a/lib/mapping.yaml +++ /dev/null @@ -1,69 +0,0 @@ -mappings: - _default_: - dynamic: 'false' - occurrences: - properties: - action_name: - type: 'string' - index: 'not_analyzed' - controller_name: - type: 'string' - index: 'not_analyzed' - project_name: - type: 'string' - index: 'not_analyzed' - uber_key: - type: 'string' - index: 'not_analyzed' - exception_class: - type: 'string' - index: 'not_analyzed' - occurred_at_day: - type: 'date' - occurred_at: - type: 'date' - exceptions: - properties: - project_name: - type: 'string' - index: 'not_analyzed' - closed: - type: 'boolean' - last_occurrence: - properties: - action_name: - type: 'string' - index: 'not_analyzed' - controller_name: - type: 'string' - index: 'not_analyzed' - project_name: - type: 'string' - index: 'not_analyzed' - uber_key: - type: 'string' - index: 'not_analyzed' - exception_class: - type: 'string' - index: 'not_analyzed' - occurred_at_day: - type: 'date' - occurred_at: - type: 'date' - first_occurred_at: - type: 'date' - occurrences_count: - type: 'long' - deploys: - properties: - project_name: - type: 'string' - index: 'not_analyzed' - version: - type: 'string' - index: 'not_analyzed' - deploy_time: - type: 'date' - changelog_link: - type: 'string' - index: 'not_analyzed' diff --git a/lib/mapping/deploys.yaml b/lib/mapping/deploys.yaml new file mode 100644 index 0000000..0a54c54 --- /dev/null +++ b/lib/mapping/deploys.yaml @@ -0,0 +1,12 @@ +properties: + project_name: + type: 'string' + index: 'not_analyzed' + version: + type: 'string' + index: 'not_analyzed' + deploy_time: + type: 'date' + changelog_link: + type: 'string' + index: 'not_analyzed' diff --git a/lib/mapping/exceptions.yaml b/lib/mapping/exceptions.yaml new file mode 100644 index 0000000..24d1221 --- /dev/null +++ b/lib/mapping/exceptions.yaml @@ -0,0 +1,10 @@ +properties: + project_name: + type: 'string' + index: 'not_analyzed' + closed: + type: 'boolean' + first_occurred_at: + type: 'date' + occurrences_count: + type: 'long' diff --git a/lib/mapping/mapping_helper.rb b/lib/mapping/mapping_helper.rb new file mode 100644 index 0000000..4594ebd --- /dev/null +++ b/lib/mapping/mapping_helper.rb @@ -0,0 +1,19 @@ +class MappingHelper + def self.get_mapping + occur = YAML.load(File.read('lib/mapping/occurrences.yaml')) + exce = YAML.load(File.read('lib/mapping/exceptions.yaml')) + deploy = YAML.load(File.read('lib/mapping/deploys.yaml')) + + exce['properties']['last_occurrence'] = occur + + { + 'mappings' => { + '_default_' => { + 'dynamic' => 'false'}, + 'occurrences' => occur, + 'exceptions' => exce, + 'deploys' => deploy + } + } + end +end diff --git a/lib/mapping/occurrences.yaml b/lib/mapping/occurrences.yaml new file mode 100644 index 0000000..9c82a76 --- /dev/null +++ b/lib/mapping/occurrences.yaml @@ -0,0 +1,20 @@ +properties: + action_name: + type: 'string' + index: 'not_analyzed' + controller_name: + type: 'string' + index: 'not_analyzed' + project_name: + type: 'string' + index: 'not_analyzed' + uber_key: + type: 'string' + index: 'not_analyzed' + exception_class: + type: 'string' + index: 'not_analyzed' + occurred_at_day: + type: 'date' + occurred_at: + type: 'date' diff --git a/test/test_helper.rb b/test/test_helper.rb index e772f4c..fef4f42 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -21,7 +21,7 @@ nodes: 1, ) -Exceptionist.esclient.create_indices('exceptionist', YAML.load(File.read('lib/mapping.yaml') )) +Exceptionist.esclient.create_indices( 'exceptionist', MappingHelper.get_mapping ) Exceptionist.esclient.refresh # Configure diff --git a/utils.rb b/utils.rb index d2947bc..6248810 100644 --- a/utils.rb +++ b/utils.rb @@ -50,7 +50,7 @@ def self.run rescue Elasticsearch::Transport::Transport::Errors::NotFound end - Exceptionist.esclient.create_indices('exceptionist',YAML.load(File.read('lib/mapping.yaml'))) + Exceptionist.esclient.create_indices('exceptionist', MappingHelper.get_mapping) Exceptionist.esclient.refresh end end From 240fd9dba49233a62108db98c8e511ec1d1e8010 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 14:41:24 +0200 Subject: [PATCH 147/293] dont change first occurrence at --- lib/models/uber_exception.rb | 2 -- test/uber_exception_test.rb | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 761fe7d..06749ab 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -41,7 +41,6 @@ def self.find_since_last_deploy(project: '', from: 0, size: 25) break end end - exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time).occurred_at end exces end @@ -61,7 +60,6 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', from: break end end - exce.first_occurred_at = Occurrence.find_next(exce.id, deploy.deploy_time).occurred_at end exces end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index f96a813..7a6a1b8 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -76,17 +76,17 @@ def test_find_since_last_deploy assert_equal [@exce1, @exce3, @exce2], uber_exces assert_equal 5, uber_exces[0].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[0].first_occurred_at + assert_equal Time.local(2011, 1, 1), uber_exces[0].first_occurred_at assert_equal 2, uber_exces[1].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[1].first_occurred_at + assert_equal Time.local(2011, 1, 2), uber_exces[1].first_occurred_at assert_equal 4, uber_exces[2].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[2].first_occurred_at + assert_equal Time.local(2011, 1, 4), uber_exces[2].first_occurred_at uber_exces = UberException.find_since_last_deploy(project: 'OtherProject') assert_equal [@exce5], uber_exces assert_equal 1, uber_exces[0].occurrences_count - assert_equal Time.local(2011, 1, 8), uber_exces[0].first_occurred_at + assert_equal Time.local(2011, 1, 4), uber_exces[0].first_occurred_at end def test_find_since_last_deploy_with_no_deploy @@ -99,11 +99,11 @@ def test_find_since_last_deploy_ordered_by_occurrences_count assert_equal [@exce1, @exce2, @exce3], uber_exces assert_equal 5, uber_exces[0].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[0].first_occurred_at + assert_equal Time.local(2011, 1, 1), uber_exces[0].first_occurred_at assert_equal 4, uber_exces[1].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[1].first_occurred_at + assert_equal Time.local(2011, 1, 4), uber_exces[1].first_occurred_at assert_equal 2, uber_exces[2].occurrences_count - assert_equal Time.local(2011, 1, 6), uber_exces[2].first_occurred_at + assert_equal Time.local(2011, 1, 2), uber_exces[2].first_occurred_at end def test_find_new_on From 8349c03eb5a02edd55f25f1f69e05629bfce0eea Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 14:50:03 +0200 Subject: [PATCH 148/293] rename deploy_time into occurred_at --- lib/app.rb | 2 +- lib/mapping/deploys.yaml | 2 +- lib/models/deploy.rb | 14 +++++--------- lib/models/uber_exception.rb | 4 ++-- test/fixtures/deploy.json | 2 +- test/fixtures/unauth_deploy.json | 2 +- test/uber_exception_test.rb | 4 ++-- 7 files changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 5f21823..e673b2a 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -50,7 +50,7 @@ class ExceptionistApp < Sinatra::Base @deploy = @current_project.last_deploy raise ArgumentError, "There is no deploy for project #{@current_project.name}" unless @deploy - @exceptions_count = UberException.count_since(project: @current_project.name, date: @deploy.deploy_time) + @exceptions_count = UberException.count_since(project: @current_project.name, date: @deploy.occurred_at) @start = params[:start] ? params[:start].to_i : 0 if params[:sort_by] && params[:sort_by] == 'frequent' diff --git a/lib/mapping/deploys.yaml b/lib/mapping/deploys.yaml index 0a54c54..8a9b1c9 100644 --- a/lib/mapping/deploys.yaml +++ b/lib/mapping/deploys.yaml @@ -5,7 +5,7 @@ properties: version: type: 'string' index: 'not_analyzed' - deploy_time: + occurred_at: type: 'date' changelog_link: type: 'string' diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 788f7de..4a49e34 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -1,7 +1,7 @@ require 'json' class Deploy - attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :deploy_time + attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :occurred_at def self.find_by_project(project) find( filters: { term: { project_name: project } } ) @@ -11,7 +11,7 @@ def self.find_last_deploy(project) find( filters: { term: { project_name: project } }, from: 0, size: 1).first end - def self.find(filters: {}, sort: { deploy_time: { order: 'desc' } }, from: 0, size: 25) + def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 Exceptionist.esclient.search_deploys( filters: filters, sort: sort, from: from, size: size ) end @@ -20,17 +20,13 @@ def self.from_json(json) Deploy.new(attr) end - def occurred_at - deploy_time.is_a?(String) ? Time.parse(deploy_time) : deploy_time - end - def initialize(attributes={}) attributes.each do |key, value| send("#{key}=", value) end - self.deploy_time ||= Time.now - self.deploy_time = Time.parse(self.deploy_time) if self.deploy_time.is_a? String + self.occurred_at ||= Time.now + self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String end def save @@ -54,7 +50,7 @@ def to_hash api_key: api_key, version: version, changelog_link: changelog_link, - deploy_time: deploy_time.is_a?(String) ? deploy_time : deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + occurred_at: occurred_at.is_a?(String) ? occurred_at : occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 06749ab..3c8a3f9 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -29,7 +29,7 @@ def self.find_sorted_by_occurrences_count(project: '', from: 0, size: 25) def self.find_since_last_deploy(project: '', from: 0, size: 25) deploy = Deploy.find_last_deploy(project) return nil unless deploy - agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') + agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] agg_exces.each { |occurr| ids << occurr['key'] } exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } }, from: from, size: size ) @@ -48,7 +48,7 @@ def self.find_since_last_deploy(project: '', from: 0, size: 25) def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', from: 0, size: 25) deploy = Deploy.find_last_deploy(project) return nil unless deploy - agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.deploy_time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') + agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] agg_exces.each { |occurr| ids << occurr['key'] } exces = Exceptionist.esclient.mget(ids: ids.slice!(from, size)) diff --git a/test/fixtures/deploy.json b/test/fixtures/deploy.json index c04ca98..e51cb05 100644 --- a/test/fixtures/deploy.json +++ b/test/fixtures/deploy.json @@ -2,5 +2,5 @@ "api_key":"SECRET_API_KEY", "version":"0.1.0", "changelog_link":"https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e", - "deploy_time":"2014-09-03T12:00:00.000+0200" + "occurred_at":"2014-09-03T12:00:00.000+0200" } diff --git a/test/fixtures/unauth_deploy.json b/test/fixtures/unauth_deploy.json index 0616c97..c3e2301 100644 --- a/test/fixtures/unauth_deploy.json +++ b/test/fixtures/unauth_deploy.json @@ -2,5 +2,5 @@ "api_key":"UNAUTH_KEY", "version":"0.1.0", "changelog_link":"https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e", - "deploy_time":"2014-09-03T12:00:00.000+0200" + "occurred_at":"2014-09-03T12:00:00.000+0200" } diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 7a6a1b8..48bedbe 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -34,9 +34,9 @@ def setup UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4), project_name: 'OtherProject')) @exce5 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject')) - create_deploy(deploy_time: Time.local(2011, 1, 5, 12, 0)) + create_deploy(occurred_at: Time.local(2011, 1, 5, 12, 0)) - create_deploy(deploy_time: Time.local(2011, 1, 6), project_name: 'OtherProject') + create_deploy(occurred_at: Time.local(2011, 1, 6), project_name: 'OtherProject') Exceptionist.esclient.refresh end From 6db5c54b3e935c32053c896e4fc9029abfd4929d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 14:57:51 +0200 Subject: [PATCH 149/293] remove whitespace --- test/uber_exception_test.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 48bedbe..d4e0dfb 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -150,7 +150,6 @@ def test_close! assert_equal [], UberException.find(filters: {term: { id: @exce1.id } } ) end - def test_current_occurrence assert_equal @occur1, @exce1.current_occurrence(1) assert_equal @occur2, @exce1.current_occurrence(2) @@ -162,7 +161,6 @@ def test_current_occurrence end end - def test_occurrences_count_on assert_equal 1, @exce1.occurrences_count_on(Time.local(2011, 1, 9)) assert_equal 3, @exce2.occurrences_count_on(Time.local(2011, 1, 4)) From d46aceb069f46d5cd7cd62a34f90019efdd59762 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 15:19:27 +0200 Subject: [PATCH 150/293] mark attr access section --- lib/models/uber_exception.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 3c8a3f9..b5e058c 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -131,6 +131,10 @@ def current_occurrence(position) occurrences.any? ? occurrences.first : nil end + # + # accessors + # + def title last_occurrence.title end From f369d32a01021b42fb368b6e0c1eb47761ba5a32 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 16:02:16 +0200 Subject: [PATCH 151/293] add since last deploy method --- lib/models/uber_exception.rb | 5 +++++ test/uber_exception_test.rb | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index b5e058c..ed74c73 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -131,6 +131,11 @@ def current_occurrence(position) occurrences.any? ? occurrences.first : nil end + def new_since_last_deploy + deploy = Deploy.find_last_deploy(@project_name) + deploy.nil? ? true : deploy.occurred_at < @first_occurred_at + end + # # accessors # diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index d4e0dfb..abd550b 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -34,10 +34,14 @@ def setup UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 4), project_name: 'OtherProject')) @exce5 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'OtherProject')) + @exce6 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), project_name: 'ThirdProject')) + create_deploy(occurred_at: Time.local(2011, 1, 5, 12, 0)) create_deploy(occurred_at: Time.local(2011, 1, 6), project_name: 'OtherProject') + create_deploy(occurred_at: Time.local(2011, 1, 6), project_name: 'ThirdProject') + Exceptionist.esclient.refresh end @@ -165,4 +169,10 @@ def test_occurrences_count_on assert_equal 1, @exce1.occurrences_count_on(Time.local(2011, 1, 9)) assert_equal 3, @exce2.occurrences_count_on(Time.local(2011, 1, 4)) end + + def test_new_since + assert ! @exce1.new_since_last_deploy + assert ! @exce3.new_since_last_deploy + assert @exce6.new_since_last_deploy + end end From 33d3e9f7d1f6fa25f35b96e4d1233fa0c0ecb85d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 16:03:01 +0200 Subject: [PATCH 152/293] extend test --- test/deploy_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/deploy_test.rb b/test/deploy_test.rb index 1f4a730..3a6602a 100644 --- a/test/deploy_test.rb +++ b/test/deploy_test.rb @@ -23,5 +23,7 @@ def test_save def test_find_last_deploy assert_equal @deploy13, Deploy.find_last_deploy('ExampleProject') assert_equal @deploy21, Deploy.find_last_deploy('OtherProject') + + assert_equal nil, Deploy.find_last_deploy('NoProject') end end From 8d64196c3d11c03185c35b624e33da4d1f3a3732 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 16:04:55 +0200 Subject: [PATCH 153/293] add new since last deploy --- public/stylesheets/exceptionist.css | 14 +++++++++++--- test/integration_test.rb | 18 ++++++++++++++++++ views/_exceptions.erb | 7 +++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index 0cb6819..49b2379 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -78,14 +78,22 @@ dd { text-align: right; } +span.box { + float: right; + font-size: 1.2em; + padding: 6px 8px; +} + span.occurrences_count { - float: right; background-color: #eee; - font-size: 1.2em; - padding: 6px 8px; color: #666; } +span.new { + background-color: #F0113E; + color: #ddd; +} + .project-label { background: #666; color: white; diff --git a/test/integration_test.rb b/test/integration_test.rb index bfdd54c..8f9d7a2 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -251,6 +251,24 @@ def test_projects_since_last_deploy_pagination assert_contain 'previous page' end + def test_projects_new_exception_since_deploy + create_deploy + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject' + assert_contain 'new' + + create_deploy + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject' + assert_not_contain 'new' + end + def test_exceptions_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) diff --git a/views/_exceptions.erb b/views/_exceptions.erb index 602fdcb..a024f2e 100644 --- a/views/_exceptions.erb +++ b/views/_exceptions.erb @@ -3,12 +3,15 @@ Last: <%= format_time(uber_exc.last_occurrence.occurred_at) %>
First: <%= format_time(uber_exc.first_occurred_at) %>
-
+
<%= escape_html(truncate(uber_exc.title, 100)) %>
<%= escape_html(truncate(uber_exc.url, 80)) %> +
+
+  <% if uber_exc.new_since_last_deploy %> new <% end %>
- # <%= uber_exc.occurrences_count %> + # <%= uber_exc.occurrences_count %>
<% end %> From 0dac15307eb2a3ff8e45b51df81845fc0122a607 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 17 Sep 2014 16:17:37 +0200 Subject: [PATCH 154/293] add category field --- lib/mapping/exceptions.yaml | 3 +++ lib/models/uber_exception.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/mapping/exceptions.yaml b/lib/mapping/exceptions.yaml index 24d1221..290b2cb 100644 --- a/lib/mapping/exceptions.yaml +++ b/lib/mapping/exceptions.yaml @@ -8,3 +8,6 @@ properties: type: 'date' occurrences_count: type: 'long' + category: + type: 'string' + index: 'not_analyzed' diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index ed74c73..c53816e 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,5 +1,5 @@ class UberException - attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at + attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at, :category def initialize(attributes) @id = attributes[:id] From ba898ab4f4bff6131ffbaa7d841dee7f757b514b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 08:52:05 +0200 Subject: [PATCH 155/293] move require json in boot.rb --- lib/boot.rb | 1 + lib/models/deploy.rb | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/boot.rb b/lib/boot.rb index 342d13d..74de874 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -8,6 +8,7 @@ require 'yajl' require 'nokogiri' require 'yaml' +require 'json' require 'models/project' require 'models/uber_exception' diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 4a49e34..4d3d1ba 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -1,5 +1,3 @@ -require 'json' - class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :occurred_at From a4858e41ad0367841bca2c79f03606997a27bab0 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 08:52:44 +0200 Subject: [PATCH 156/293] insert whitespace --- lib/models/deploy.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 4d3d1ba..c982cfc 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -13,6 +13,7 @@ def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, si raise ArgumentError, 'position has to be >= 0' if from < 0 Exceptionist.esclient.search_deploys( filters: filters, sort: sort, from: from, size: size ) end + def self.from_json(json) attr = symbolize_keys(JSON.parse(json)) Deploy.new(attr) From 71a84004ef9a77ed1a0f53ffd00824da98858b76 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 08:53:36 +0200 Subject: [PATCH 157/293] insert occurred_at in json constructor --- lib/models/deploy.rb | 3 ++- test/test_helper.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index c982cfc..8b800fa 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -16,6 +16,8 @@ def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, si def self.from_json(json) attr = symbolize_keys(JSON.parse(json)) + attr['occurred_at'] = Time.now if attr['occurred_at'].nil? + Deploy.new(attr) end @@ -24,7 +26,6 @@ def initialize(attributes={}) send("#{key}=", value) end - self.occurred_at ||= Time.now self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String end diff --git a/test/test_helper.rb b/test/test_helper.rb index fef4f42..6864b48 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -57,7 +57,8 @@ def build_deploy(attributes = {}) project_name: 'ExampleProject', api_key: 'SECRET_API_KEY', version: '0.0.1', - changelog_link: 'https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e' + changelog_link: 'https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e', + occurred_at: Time.now } Deploy.new(default_attributes.merge(attributes)) end From 247994ccf8429be0937823a72505938ed0fbb3c3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:02:45 +0200 Subject: [PATCH 158/293] insert occurred_at in xml constructor --- lib/models/occurrence.rb | 8 +++++--- test/test_helper.rb | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 95a8db7..463b125 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -10,8 +10,7 @@ def initialize(attributes={}) send("#{key}=", value) end - self.occurred_at = Time.parse(attributes['occurred_at']) if attributes['occurred_at'] - self.occurred_at ||= Time.now + self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String self.uber_key ||= generate_uber_key end @@ -136,7 +135,10 @@ def to_hash end def self.from_xml(xml_text) - new(parse_xml(xml_text)) + attr = parse_xml(xml_text) + attr['occurred_at'] = Time.now if attr['occurred_at'].nil? + + new(attr) end def self.parse_xml(xml_text) diff --git a/test/test_helper.rb b/test/test_helper.rb index 6864b48..9fcb4cc 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -43,7 +43,8 @@ def build_occurrence(attributes = {}) controller_name: 'users', action_name: 'show', project_name: 'ExampleProject', - url: 'http://example.com' + url: 'http://example.com', + occurred_at: Time.now } Occurrence.new(default_attributes.merge(attributes)) end From 03a58d7a2f6eafc1428086530f0389050d8b57cf Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:03:35 +0200 Subject: [PATCH 159/293] indent lines --- test/test_helper.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 9fcb4cc..3164dfc 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -55,11 +55,11 @@ def create_occurrence(attributes = {}) def build_deploy(attributes = {}) default_attributes = { - project_name: 'ExampleProject', - api_key: 'SECRET_API_KEY', - version: '0.0.1', - changelog_link: 'https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e', - occurred_at: Time.now + project_name: 'ExampleProject', + api_key: 'SECRET_API_KEY', + version: '0.0.1', + changelog_link: 'https://github.com/podio/podio-rb/commit/35b1bbaaafd56b200ee4a0ea38fc13dfdea8304e', + occurred_at: Time.now } Deploy.new(default_attributes.merge(attributes)) end From 06fc19f34cbdc7fc90044e5bbd694ff484514a8e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:10:24 +0200 Subject: [PATCH 160/293] return only aggregation results --- lib/es_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 6a9e1af..d12eb6f 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -36,7 +36,7 @@ def search_deploys(filters: {}, sort: {}, from: 0, size: 25) def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } - response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query) + response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query, search_type: 'count') hash = Hashie::Mash.new(response) hash.aggregations.exceptions.buckets end From f933fe9b1d1959d312c3bdea6aeb97ce50378888 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:13:10 +0200 Subject: [PATCH 161/293] move constructor to the top --- lib/models/deploy.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 8b800fa..1cac4e4 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -1,6 +1,14 @@ class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :occurred_at + def initialize(attributes={}) + attributes.each do |key, value| + send("#{key}=", value) + end + + self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String + end + def self.find_by_project(project) find( filters: { term: { project_name: project } } ) end @@ -21,14 +29,6 @@ def self.from_json(json) Deploy.new(attr) end - def initialize(attributes={}) - attributes.each do |key, value| - send("#{key}=", value) - end - - self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String - end - def save deploy = Exceptionist.esclient.index('deploys', to_hash) @id = deploy._id From ebe59a05b6f3e8825ef11c07248d4cad7681f984 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:35:18 +0200 Subject: [PATCH 162/293] use @ instead of self --- lib/models/deploy.rb | 2 +- lib/models/occurrence.rb | 4 ++-- lib/models/project.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 1cac4e4..0557ea5 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -6,7 +6,7 @@ def initialize(attributes={}) send("#{key}=", value) end - self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String + @occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String end def self.find_by_project(project) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 463b125..f082d08 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -10,8 +10,8 @@ def initialize(attributes={}) send("#{key}=", value) end - self.occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String - self.uber_key ||= generate_uber_key + @occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String + @uber_key ||= generate_uber_key end def uber_exception diff --git a/lib/models/project.rb b/lib/models/project.rb index 00744bd..a4bc41f 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -2,7 +2,7 @@ class Project attr_accessor :name def initialize(name) - self.name = name + @name = name end def exceptions_count From 8e37b9e5f2c1659fc33bccf08d3755b4be8de717 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:35:56 +0200 Subject: [PATCH 163/293] dynamic initialisation --- lib/models/uber_exception.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index c53816e..5ed7d8e 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -2,12 +2,12 @@ class UberException attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at, :category def initialize(attributes) - @id = attributes[:id] - @project_name = attributes[:project_name] - @occurrences_count = attributes[:occurrences_count] - @closed = attributes[:closed] + attributes.each do |key, value| + instance_variable_set("@#{key}", value) + end + @last_occurrence = Occurrence.new(attributes[:last_occurrence]) - @first_occurred_at = Time.parse(attributes[:first_occurred_at]) + @first_occurred_at = Time.parse(self.first_occurred_at) if self.first_occurred_at.is_a? String end def self.count_all(project) From 6360374388cea14adb13cde53da590b4ae81c1b4 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:39:53 +0200 Subject: [PATCH 164/293] proper function, avoid errors with unkown vars --- lib/models/deploy.rb | 2 +- lib/models/occurrence.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 0557ea5..c73bc91 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -3,7 +3,7 @@ class Deploy def initialize(attributes={}) attributes.each do |key, value| - send("#{key}=", value) + instance_variable_set("@#{key}", value) end @occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index f082d08..2616cd3 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -7,7 +7,7 @@ class Occurrence def initialize(attributes={}) attributes.each do |key, value| - send("#{key}=", value) + instance_variable_set("@#{key}", value) end @occurred_at = Time.parse(self.occurred_at) if self.occurred_at.is_a? String From 34e0de3a6a1ae955d4dace37044848ea96afbd6d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:41:35 +0200 Subject: [PATCH 165/293] dont need do deleted anymore --- lib/es_client.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index d12eb6f..32c2642 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -118,10 +118,6 @@ def transform(attr) attr.merge!(attr['_source']).delete('_source') attr = symbolize_keys(attr) attr[:id] = attr.delete :_id - attr.delete :_index - attr.delete :_type - attr.delete :_score - attr.delete :sort attr end From 186d6034996e1b808f875f0a0641c29eacf29e0f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 09:52:10 +0200 Subject: [PATCH 166/293] remove superfluo div --- views/_occurrence.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 5c9d403..2f8b6f7 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -21,7 +21,6 @@
<%= @occurrence.http_method %> <%= @occurrence.url %>
-
<%= format_time(@occurrence.occurred_at) %>
From 2e007ae30b356d147cd41dcebe1a8f54a50b48cc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 10:28:53 +0200 Subject: [PATCH 167/293] create update function for uber_exception --- lib/models/uber_exception.rb | 4 ++++ test/uber_exception_test.rb | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 5ed7d8e..5d4d979 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -126,6 +126,10 @@ def close! Exceptionist.esclient.update('exceptions', @id, { doc: { closed: true } }) end + def update(doc) + Exceptionist.esclient.update('exceptions', @id, { doc: doc }) + end + def current_occurrence(position) occurrences = Occurrence.find(uber_key: id, sort: { occurred_at: { order: 'asc'} }, from: position - 1, size: 1) occurrences.any? ? occurrences.first : nil diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index abd550b..a8fbc65 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -175,4 +175,13 @@ def test_new_since assert ! @exce3.new_since_last_deploy assert @exce6.new_since_last_deploy end + + def test_update + assert_equal nil, @exce1.category + + @exce1.update( { category: "low" } ) + Exceptionist.esclient.refresh + + assert_equal "low", UberException.get(@exce1.id).category + end end From 3849ef5f91d444349db6e56117d02e0871433270 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 10:29:38 +0200 Subject: [PATCH 168/293] enable category on homepage --- lib/app.rb | 7 +++++++ public/stylesheets/exceptionist.css | 17 +++++++++++++++-- views/_exceptions.erb | 20 ++++++++++++-------- views/_occurrence.erb | 11 +++++++++++ views/index.erb | 1 - 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index e673b2a..cd47f8d 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -119,6 +119,13 @@ class ExceptionistApp < Sinatra::Base redirect to("/projects/#{@uber_exceptions.project_name}?#{Rack::Utils.unescape(params[:backparams])}") end + post '/exceptions/:id/category' do + @uber_exceptions = UberException.get(params[:id]) + @uber_exceptions.update( category: params[:category] ) + + redirect to("/projects/#{@uber_exceptions.project_name}?#{Rack::Utils.unescape(params[:backparams])}") + end + post '/notifier_api/v2/notices/?' do occurrence = Occurrence.from_xml(params[:data] || request.body.read) project = Project.find_by_key(occurrence.api_key) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index 49b2379..7e277d1 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -85,13 +85,14 @@ span.box { } span.occurrences_count { - background-color: #eee; - color: #666; + background-color: #ddd; + color: #777; } span.new { background-color: #F0113E; color: #ddd; + margin-right: 10px; } .project-label { @@ -105,6 +106,18 @@ span.new { margin-bottom:0.5em; } +.low { + background-color: aliceblue; +} + +.high { + background-color: antiquewhite; +} + +.padding { + padding: 0 5px; +} + /* -------------------------------------------------------------- diff --git a/views/_exceptions.erb b/views/_exceptions.erb index a024f2e..7a4e2c2 100644 --- a/views/_exceptions.erb +++ b/views/_exceptions.erb @@ -1,17 +1,21 @@ <% for uber_exc in exceptions %> +
+
- Last: <%= format_time(uber_exc.last_occurrence.occurred_at) %>
- First: <%= format_time(uber_exc.first_occurred_at) %> +
+ Last: <%= format_time(uber_exc.last_occurrence.occurred_at) %>
+ First: <%= format_time(uber_exc.first_occurred_at) %> +
<%= escape_html(truncate(uber_exc.title, 100)) %>
<%= escape_html(truncate(uber_exc.url, 80)) %>
-
-  <% if uber_exc.new_since_last_deploy %> new <% end %> -
-
- # <%= uber_exc.occurrences_count %> +
+
+ # <%= uber_exc.occurrences_count %> + <% if uber_exc.new_since_last_deploy %> new <% end %> +
-
+
<% end %> diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 2f8b6f7..3fde9ac 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -19,6 +19,17 @@
<%= format_time(@uber_exception.last_occurrence.occurred_at) %>
+
+ + + + + +
+
<%= @occurrence.http_method %> <%= @occurrence.url %>
diff --git a/views/index.erb b/views/index.erb index e2f70f9..4f9b85b 100644 --- a/views/index.erb +++ b/views/index.erb @@ -12,7 +12,6 @@
-
<%= partial(:exceptions, :exceptions => @uber_exceptions) %>
From b4ef5f82f3db47ff082da92b342e0c3954698f6e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 10:47:25 +0200 Subject: [PATCH 169/293] add whitespace --- public/stylesheets/exceptionist.css | 2 +- views/_occurrence.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index 7e277d1..a554079 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -103,7 +103,7 @@ span.new { } .append-tiny-bottom { - margin-bottom:0.5em; + margin-bottom: 0.5em; } .low { diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 3fde9ac..e938708 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -22,7 +22,7 @@
- From aa9bc6d2f84c0442caa013f5daaae0f51248845b Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 13:03:15 +0200 Subject: [PATCH 170/293] test updating category --- test/integration_test.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index 8f9d7a2..f90688b 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -312,6 +312,34 @@ def test_exceptions_paginate_occurrences assert_not_contain 'GET http://example.com/?show=two' end + def test_exceptions_change_category + occur1 = create_occurrence(url: 'http://example.com/?show=one') + UberException.occurred(occur1) + + Exceptionist.esclient.refresh + + visit "/exceptions/#{occur1.uber_key}" + assert_contain 'low' + assert_contain 'high' + + select 'high' + click_button 'save' + Exceptionist.esclient.refresh + follow_redirect! + + assert_equal 'http://example.org/projects/ExampleProject', last_request.url + assert_have_selector '.high' + + visit "/exceptions/#{occur1.uber_key}" + + select 'low' + click_button 'save' + Exceptionist.esclient.refresh + follow_redirect! + + assert_have_selector '.low' + end + def test_projects_be_able_to_close_an_exception UberException.occurred(create_occurrence(action_name:'show')) UberException.occurred(create_occurrence(action_name:'index')) From fa4ea8ab8cb32dfd9e483afe88f2fe73efc09eed Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 13:16:57 +0200 Subject: [PATCH 171/293] show default value in select --- views/_occurrence.erb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/views/_occurrence.erb b/views/_occurrence.erb index e938708..5ba0430 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -23,8 +23,9 @@ From e2f86abe50a73d95ffe048343923a09c1621ce87 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Thu, 18 Sep 2014 13:19:00 +0200 Subject: [PATCH 172/293] use submit form instead of button click --- test/integration_test.rb | 6 +++--- views/_occurrence.erb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index f90688b..066b166 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -323,7 +323,7 @@ def test_exceptions_change_category assert_contain 'high' select 'high' - click_button 'save' + submit_form 'category' Exceptionist.esclient.refresh follow_redirect! @@ -333,7 +333,7 @@ def test_exceptions_change_category visit "/exceptions/#{occur1.uber_key}" select 'low' - click_button 'save' + submit_form 'category' Exceptionist.esclient.refresh follow_redirect! @@ -352,7 +352,7 @@ def test_projects_be_able_to_close_an_exception click_link 'NameError in users#show' - click_button 'Close' + submit_form 'close' Exceptionist.esclient.refresh diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 5ba0430..5f0be22 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -20,7 +20,7 @@
-
+
<% end %> diff --git a/views/show.erb b/views/show.erb index ec302d4..9fd3c61 100644 --- a/views/show.erb +++ b/views/show.erb @@ -16,7 +16,7 @@
- <%= partial(:thirty_day_graph, :holder_div_id => 'occurrence_graph', :data => @uber_exception.last_thirty_days) %> + <%= partial(:thirty_day_graph, :holder_div_id => 'occurrence_graph', :data => @uber_exception.last_thirty_days, :deploys => @current_project.deploys_last_thirty_days) %>
From 732c686a2f1fad7156f2ca649816bd6feb60efca Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 10:52:20 +0200 Subject: [PATCH 178/293] set configuration vars for test enviornment --- lib/app.rb | 6 ++++++ test/integration_test.rb | 2 ++ test/test_helper.rb | 2 ++ 3 files changed, 10 insertions(+) diff --git a/lib/app.rb b/lib/app.rb index cd47f8d..5c449d2 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -11,6 +11,12 @@ class ExceptionistApp < Sinatra::Base set :views, "#{dir}/views" set :public_folder, "#{dir}/public" + configure :test do + set :raise_errors, true + set :dump_errors, false + set :show_exceptions, false + end + before do protected! if request.path_info !~ /^\/notifier_api\/v2/ end diff --git a/test/integration_test.rb b/test/integration_test.rb index 066b166..135ea70 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -1,3 +1,5 @@ +ENV['RACK_ENV'] = 'test' + require 'test_helper' require 'rack/test' diff --git a/test/test_helper.rb b/test/test_helper.rb index 9a818e3..b71a9d2 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,6 +4,8 @@ add_filter 'test' end +ENV['RACK_ENV'] = 'test' + require 'app' require 'elasticsearch/extensions/test/cluster' From 7f0341adfd184c1281b8575164c1089e29327c54 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 10:54:31 +0200 Subject: [PATCH 179/293] sinatra now throws errors, add catch block --- test/integration_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 135ea70..803bf1f 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -184,9 +184,9 @@ def test_projects_forget_old_exceptions end def test_projects_since_last_deploy_with_no_deploy - get '/projects/ExampleProject/since_last_deploy' - - assert !last_response.ok? + assert_raises(ArgumentError) do + get '/projects/ExampleProject/since_last_deploy' + end end def test_projects_since_last_deploy_fresh_deploy From 5bfbc41727775e5868e727849a172d0d22f903f7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 10:55:04 +0200 Subject: [PATCH 180/293] formatting, move lib to the other libs --- test/test_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index b71a9d2..2eb7e18 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,4 +1,3 @@ -require 'timecop' require 'simplecov' SimpleCov.start do add_filter 'test' @@ -7,6 +6,7 @@ ENV['RACK_ENV'] = 'test' require 'app' +require 'timecop' require 'elasticsearch/extensions/test/cluster' From 03c9c729cde5758fd95b9f46a5365f6f7b8c43c1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 11:40:12 +0200 Subject: [PATCH 181/293] refactor find uber_exception --- lib/app.rb | 2 +- lib/models/uber_exception.rb | 12 ++++++------ test/api_test.rb | 22 +++++++++++----------- test/uber_exception_test.rb | 12 ++++++------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 5c449d2..ce029bc 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -43,7 +43,7 @@ class ExceptionistApp < Sinatra::Base if params[:sort_by] && params[:sort_by] == 'frequent' @uber_exceptions = UberException.find_sorted_by_occurrences_count(project: @current_project.name, from: @start) else - @uber_exceptions = UberException.find(project: @current_project.name, from: @start) + @uber_exceptions = UberException.find(terms: [ { project_name: @current_project.name } ], from: @start) end @title = "Latest Exceptions for #{@current_project.name}" erb :index diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 5d4d979..f449c9a 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -23,7 +23,7 @@ def self.get(uber_key) end def self.find_sorted_by_occurrences_count(project: '', from: 0, size: 25) - UberException.find(project: project, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) + UberException.find(terms: [ { project_name: project } ], sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end def self.find_since_last_deploy(project: '', from: 0, size: 25) @@ -32,7 +32,7 @@ def self.find_since_last_deploy(project: '', from: 0, size: 25) agg_exces = Exceptionist.esclient.search_aggs([ { term: { project_name: project } }, { range: { occurred_at: { gte: deploy.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ],'uber_key') ids = [] agg_exces.each { |occurr| ids << occurr['key'] } - exces = find( project: project, filters: { ids: { type: 'exceptions', values: ids } }, from: from, size: size ) + exces = find( terms: [ { project_name: project } ], filters: [ { ids: { type: 'exceptions', values: ids } } ], from: from, size: size ) exces.each do |exce| agg_exces.each do |occurr| if occurr['key'] == exce.id @@ -64,12 +64,12 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', from: exces end - def self.find(project: '', filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) + def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 - filters = [filters] if filters.class == Hash - filters << { term: { closed: false } } << { term: { project_name: project } } - Exceptionist.esclient.search_exceptions(filters: filters, sort: sort, from: from, size: size) + terms = terms.map { |term| { term: term } } + terms << { term: { closed: false } } + Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) end def self.find_new_on(project, day) diff --git a/test/api_test.rb b/test/api_test.rb index b565e9f..b9a7de7 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -14,14 +14,14 @@ def app end def test_create_the_first_UberException - assert_equal [], UberException.find( project: 'ExampleProject' ) + assert_equal [], UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find( project: 'ExampleProject' ) + exce = UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end @@ -32,14 +32,14 @@ def test_add_occurrences_if_it_is_the_same_exception Exceptionist.esclient.refresh - assert_equal 1, UberException.find( project: 'ExampleProject' ).count + assert_equal 1, UberException.find( terms: [ { project_name: 'ExampleProject' } ] ).count post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find( project: 'ExampleProject' ) + exce = UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) assert_equal 1, exce.count assert_equal 2, exce.first.occurrences_count end @@ -52,44 +52,44 @@ def test_api_unauth_exception Exceptionist.esclient.refresh - assert_equal [], UberException.find( project: 'ExampleProject' ) + assert_equal [], UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) end def test_api_full_exception - assert_equal [], UberException.find( project: 'ExampleProject' ) + assert_equal [], UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/full_exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find( project: 'ExampleProject' ) + exce = UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end def test_api_exception_with_hash - assert_equal [], UberException.find( project: 'ExampleProject' ) + assert_equal [], UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/exception_with_hash_in_params.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find( project: 'ExampleProject' ) + exce = UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end def test_api_minimal_exception - assert_equal [], UberException.find( project: 'ExampleProject' ) + assert_equal [], UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) post '/notifier_api/v2/notices/', read_fixtures_file('fixtures/minimal_exception.xml') assert last_response.ok? Exceptionist.esclient.refresh - exce = UberException.find( project: 'ExampleProject' ) + exce = UberException.find( terms: [ { project_name: 'ExampleProject' } ] ) assert_equal 1, exce.count assert_equal 1, exce.first.occurrences_count end diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index a8fbc65..b534c30 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -66,8 +66,8 @@ def test_get end def test_find - assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(project: 'ExampleProject') - assert_equal [@exce2], UberException.find(project: 'ExampleProject', from: 2, size: 1) + assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(terms: [ { project_name: 'ExampleProject' } ] ) + assert_equal [@exce2], UberException.find(terms: [ { project_name: 'ExampleProject' } ], from: 2, size: 1) end def test_find_sorted_by_occurrences_count @@ -123,18 +123,18 @@ def test_forget_old_exceptions UberException.forget_old_exceptions('ExampleProject', days) Exceptionist.esclient.refresh - assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(project: 'ExampleProject') + assert_equal [@exce1, @exce3, @exce2, @exce4], UberException.find(terms: [ { project_name: 'ExampleProject' } ]) days = (Time.now - Time.local(2011, 1, 7)) / 86400 UberException.forget_old_exceptions('ExampleProject', days) Exceptionist.esclient.refresh - assert_equal [@exce1, @exce3], UberException.find(project: 'ExampleProject') + assert_equal [@exce1, @exce3], UberException.find(terms: [ { project_name: 'ExampleProject' } ]) UberException.forget_old_exceptions('ExampleProject') Exceptionist.esclient.refresh - assert_equal [], UberException.find(project: 'ExampleProject') + assert_equal [], UberException.find(terms: [ { project_name: 'ExampleProject' } ]) end def test_forget! @@ -151,7 +151,7 @@ def test_close! Exceptionist.esclient.refresh - assert_equal [], UberException.find(filters: {term: { id: @exce1.id } } ) + assert_equal [], UberException.find(terms: [ { id: @exce1.id } ] ) end def test_current_occurrence From edaa3ea7eb1f796b25aacaf75c1daa5dd9a79a35 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 13:30:50 +0200 Subject: [PATCH 182/293] change message --- test/integration_test.rb | 4 ++-- views/index.erb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 803bf1f..2422f16 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -97,7 +97,7 @@ def test_projects_with_no_exceptions visit '/projects/ExampleProject' assert_contain 'Latest Exceptions for ExampleProject' - assert_contain 'no exceptions' + assert_contain 'No exceptions' assert_not_contain 'next page' assert_not_contain 'previous page' end @@ -200,7 +200,7 @@ def test_projects_since_last_deploy_fresh_deploy visit '/projects/ExampleProject/since_last_deploy' assert_contain 'since last deploy' - assert_contain 'no exceptions' + assert_contain 'No exceptions' end def test_projects_since_last_deploy_old_deploy diff --git a/views/index.erb b/views/index.erb index 4f9b85b..aafc7f3 100644 --- a/views/index.erb +++ b/views/index.erb @@ -28,7 +28,7 @@
<% else %> -

Are you kidding? There are no exceptions!

+

No exceptions!

<% end %>
From 7e5a992ae44a115c99f0efbe3c2ace265d7bbc27 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 13:33:01 +0200 Subject: [PATCH 183/293] enable filtering on frontend --- lib/app.rb | 8 ++++-- lib/models/uber_exception.rb | 8 +++--- test/integration_test.rb | 51 ++++++++++++++++++++++++++++++++++++ test/uber_exception_test.rb | 4 +-- views/index.erb | 5 ++++ 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index ce029bc..55e25dc 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -39,11 +39,15 @@ class ExceptionistApp < Sinatra::Base @projects = Project.all @current_project = Project.new(params[:project]) @exceptions_count = UberException.count_all(@current_project.name) + @start = params[:start] ? params[:start].to_i : 0 + @category = { category: params[:category] } if params[:category] + @terms =[ { project_name: @current_project.name } ] << @category + if params[:sort_by] && params[:sort_by] == 'frequent' - @uber_exceptions = UberException.find_sorted_by_occurrences_count(project: @current_project.name, from: @start) + @uber_exceptions = UberException.find_sorted_by_occurrences_count(terms: @terms, from: @start) else - @uber_exceptions = UberException.find(terms: [ { project_name: @current_project.name } ], from: @start) + @uber_exceptions = UberException.find(terms: @terms, from: @start) end @title = "Latest Exceptions for #{@current_project.name}" erb :index diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index f449c9a..d3cadb3 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -22,8 +22,8 @@ def self.get(uber_key) Exceptionist.esclient.get_exception(uber_key) end - def self.find_sorted_by_occurrences_count(project: '', from: 0, size: 25) - UberException.find(terms: [ { project_name: project } ], sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) + def self.find_sorted_by_occurrences_count(terms: [], from: 0, size: 25) + UberException.find(terms: terms, sort: { occurrences_count: { order: 'desc'} }, from: from, size: size) end def self.find_since_last_deploy(project: '', from: 0, size: 25) @@ -67,7 +67,7 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', from: def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 - terms = terms.map { |term| { term: term } } + terms = terms.map { |term| { term: term } unless term.nil? } terms << { term: { closed: false } } Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) end @@ -97,7 +97,7 @@ def self.occurred(occurrence) hash = occurrence.to_hash hash[:id] = occurrence.id Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', - upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1}, + upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1, category: 'no-category'}, params: { occurrence: hash, timestamp: first_timestamp} }) Exceptionist.esclient.get_exception(occurrence.uber_key) end diff --git a/test/integration_test.rb b/test/integration_test.rb index 2422f16..68df425 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -271,6 +271,57 @@ def test_projects_new_exception_since_deploy assert_not_contain 'new' end + def test_projects_filter + UberException.occurred(create_occurrence) + + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject' + assert_have_selector '.no-category' + assert_have_no_selector '.low' + assert_have_no_selector '.high' + + visit '/projects/ExampleProject?category=no-category' + assert_not_contain 'No exceptions' + + visit '/projects/ExampleProject?category=high' + assert_contain 'No exceptions' + end + + def test_projects_filter_change + @uber_exce = UberException.occurred(create_occurrence) + @uber_exce.update(category: 'high') + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject' + assert_have_selector '.high' + assert_have_no_selector '.low' + assert_have_no_selector '.no-category' + + visit '/projects/ExampleProject?category=low' + assert_contain 'No exceptions' + + visit '/projects/ExampleProject?category=high' + assert_not_contain 'No exceptions' + end + + def test_projects_sort_most_frequent_filter + @uber_exce = UberException.occurred(create_occurrence) + @uber_exce.update(category: 'low') + Exceptionist.esclient.refresh + + visit '/projects/ExampleProject?sort_by=frequent' + assert_have_selector '.low' + assert_have_no_selector '.high' + assert_have_no_selector '.no-category' + + visit '/projects/ExampleProject?category=low&sort_by=frequent' + assert_not_contain 'No exceptions' + + visit '/projects/ExampleProject?category=high&sort_by=frequent' + assert_contain 'No exceptions' + end + def test_exceptions_show_a_minimal_occurrence occurrence = create_occurrence UberException.occurred(occurrence) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index b534c30..6d3d608 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -71,7 +71,7 @@ def test_find end def test_find_sorted_by_occurrences_count - assert_equal [@exce1, @exce2, @exce3, @exce4], UberException.find_sorted_by_occurrences_count(project: 'ExampleProject') + assert_equal [@exce1, @exce2, @exce3, @exce4], UberException.find_sorted_by_occurrences_count(terms: [ { project_name: 'ExampleProject' } ] ) end def test_find_since_last_deploy @@ -177,7 +177,7 @@ def test_new_since end def test_update - assert_equal nil, @exce1.category + assert_equal 'no-category', @exce1.category @exce1.update( { category: "low" } ) Exceptionist.esclient.refresh diff --git a/views/index.erb b/views/index.erb index aafc7f3..4ca8ff7 100644 --- a/views/index.erb +++ b/views/index.erb @@ -7,6 +7,11 @@
+ <%= '↓ ' if params[:category].nil? %>all · + <%= '↓ ' if params[:category] == 'high' %>high · + <%= '↓ ' if params[:category] == 'low' %>low · + <%= '↓ ' if params[:category] == 'no-category' %>no category +  |  <%= '↓ ' if params[:sort_by].nil? || params[:sort_by] == 'latest' %>most recent · <%= '↓ ' if params[:sort_by] == 'frequent' %>most frequent
From 0529c5598aaa186fd144e1c84f303acec9ce56f3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 14:19:35 +0200 Subject: [PATCH 184/293] move pp into boot.rb --- lib/app.rb | 2 -- lib/boot.rb | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 55e25dc..340cefa 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -3,8 +3,6 @@ require 'sinatra/base' require 'net/smtp' require 'stringio' -require 'pp' - class ExceptionistApp < Sinatra::Base dir = File.join(File.dirname(__FILE__), '..') diff --git a/lib/boot.rb b/lib/boot.rb index a32ea7a..8a6b2d7 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -9,6 +9,7 @@ require 'nokogiri' require 'yaml' require 'json' +require 'pp' require 'models/project' require 'models/uber_exception' From 601cb35223f9b2e36178255eddf514a87ae7644a Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 14:19:52 +0200 Subject: [PATCH 185/293] improve test --- test/api_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api_test.rb b/test/api_test.rb index b9a7de7..6ff5039 100644 --- a/test/api_test.rb +++ b/test/api_test.rb @@ -102,7 +102,7 @@ def test_api_deploy Exceptionist.esclient.refresh - assert_equal 1, Deploy.find_by_project('ExampleProject').count + assert_equal 'ExampleProject', Deploy.find_by_project('ExampleProject').first.project_name end def test_api_unauth_deploy From 43a699043aa18b80f2e6799555f35ee21ab072a7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Fri, 19 Sep 2014 14:41:30 +0200 Subject: [PATCH 186/293] use bar instead of circle for deploy --- views/_thirty_day_graph.erb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/views/_thirty_day_graph.erb b/views/_thirty_day_graph.erb index 252455f..643e57c 100644 --- a/views/_thirty_day_graph.erb +++ b/views/_thirty_day_graph.erb @@ -1,13 +1,18 @@
From 63972194c53738fdd4681d193d00b5c1e7b2b195 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 22 Sep 2014 21:04:36 +0200 Subject: [PATCH 193/293] remove hide/show project feature --- views/dashboard.erb | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/views/dashboard.erb b/views/dashboard.erb index 42fa3a6..8c71241 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -1,12 +1,5 @@

<%= @title %>

-
- <% for project in @projects %> - - <% end %> -
- - <% for project in @projects %>

<%= project.name %> (<%= project.exceptions_count %>) @@ -17,10 +10,3 @@

<% end %> - - - From 39dad5835066584299579761005ade0621186bbc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 22 Sep 2014 21:11:31 +0200 Subject: [PATCH 194/293] use config.rb to set elasticsearch endpoint --- utils.rb | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/utils.rb b/utils.rb index 97c5b05..a045715 100644 --- a/utils.rb +++ b/utils.rb @@ -1,5 +1,6 @@ -require 'elasticsearch' require 'boot' +require 'config' +require 'elasticsearch' module Utils @@ -16,8 +17,6 @@ def self.run class Importer def self.run - Exceptionist.endpoint = 'localhost:9200' - puts "importing deploy.yaml" YAML.load(File.read('import/deploy.yaml')).each { |key, value| Deploy.new(value).save} @@ -42,8 +41,6 @@ def self.run class ClearDB def self.run - Exceptionist.endpoint = 'localhost:9200' - begin Exceptionist.esclient.delete_indices('exceptionist') rescue Elasticsearch::Transport::Transport::Errors::NotFound @@ -56,17 +53,9 @@ def self.run class Mapping def self.run - Net::HTTP.start('exceptionist.nextpodio.dk', 6000) do |http| - request = Net::HTTP::Post.new('/notifier_api/v2/deploy/?') - request.body = JSON.generate({ - 'project_name' => 'project', - 'api_key' => 'api_key', - 'changelog_link' => 'changes', - 'version' => 'version', - - }) - puts response = http.request(request) - end + pp Exceptionist.esclient.get_mapping 'deploys' + pp Exceptionist.esclient.get_mapping 'exceptions' + pp Exceptionist.esclient.get_mapping 'occurrences' end end end From d2ff3349c797f5f8d7f5686ef8cd472f27bf0207 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Mon, 22 Sep 2014 21:13:38 +0200 Subject: [PATCH 195/293] remove change category test --- test/integration_test.rb | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index 3ae4ecd..a4034ff 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -403,34 +403,6 @@ def test_exceptions_paginate_occurrences assert_not_contain 'GET http://example.com/?show=two' end - def test_exceptions_change_category - occur1 = create_occurrence(url: 'http://example.com/?show=one') - UberException.occurred(occur1) - - Exceptionist.esclient.refresh - - visit "/exceptions/#{occur1.uber_key}" - assert_contain 'low' - assert_contain 'high' - - select 'high' - submit_form 'category' - Exceptionist.esclient.refresh - follow_redirect! - - assert_equal 'http://example.org/projects/ExampleProject', last_request.url - assert_have_selector '.high' - - visit "/exceptions/#{occur1.uber_key}" - - select 'low' - submit_form 'category' - Exceptionist.esclient.refresh - follow_redirect! - - assert_have_selector '.low' - end - def test_projects_be_able_to_close_an_exception UberException.occurred(create_occurrence(action_name:'show')) UberException.occurred(create_occurrence(action_name:'index')) From 2a2da2fcd2fe910c605bd342720f38c8585c9188 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 10:38:47 +0200 Subject: [PATCH 196/293] refactoring integration tests --- test/integration_test.rb | 297 +++++++++------------------------------ test/test_helper.rb | 4 +- 2 files changed, 73 insertions(+), 228 deletions(-) diff --git a/test/integration_test.rb b/test/integration_test.rb index a4034ff..c63164c 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -16,6 +16,23 @@ class IntegrationTest < MiniTest::Test def setup clear_collections + + @exce1 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 1))) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 10))) + + @exce2 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'save')) + @exce2.update(category: 'high') + + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 6), action_name: 'delete', url: 'http://example.com/?show=one')) + UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 7), action_name: 'delete', url: 'http://example.com/?show=two')) + @exce3 = UberException.occurred(create_occurrence(occurred_at: Time.local(2011, 1, 8), action_name: 'delete', url: 'http://example.com/?show=three')) + @exce3.update(category: 'low') + + create_deploy(occurred_at: Time.local(2011, 1, 5)) + create_deploy(occurred_at: Time.local(2011, 1, 6), project_name: 'OtherProject') + create_deploy(occurred_at: Time.local(2011, 1, 6), project_name: 'ThirdProject') + + Exceptionist.esclient.refresh end def app @@ -26,103 +43,59 @@ def test_dashboard_empty visit '/' end - def test_dashboard_with_one_project - occurrence = create_occurrence - UberException.occurred(occurrence) - - Exceptionist.esclient.refresh - + def test_dashboard_contain_projects visit '/' assert_contain 'ExampleProject' + assert_contain 'ThirdProject' click_link 'ExampleProject' end - def test_dashboard_with_no_deploy - occurrence = create_occurrence - UberException.occurred(occurrence) - - Exceptionist.esclient.refresh - + def test_dashboard_contain_no_deploy_message visit '/' assert_contain '- no deploy found' end - def test_dashboard_with_deploy - occurrence = create_occurrence - UberException.occurred(occurrence) - create_deploy - - Exceptionist.esclient.refresh - + def test_dashboard_contain_deploy_message visit '/' assert_contain '- deploy:' end - def test_dashboard_with_two_projects - occur1 = create_occurrence(project_name: 'ExampleProject') - UberException.occurred(occur1) - - occur2 = create_occurrence(project_name: 'ExampleProject2') - UberException.occurred(occur2) - - Exceptionist.esclient.refresh - - visit '/' - assert_contain 'ExampleProject' - assert_contain 'ExampleProject2' - end - def test_river - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - visit '/river' assert_contain 'River' end def test_river_project - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - visit 'projects/ExampleProject/river' assert_contain 'Latest Occurrences' end def test_projects_with_no_exceptions - visit '/projects/ExampleProject' + visit '/projects/PhantomProject' - assert_contain 'Latest Exceptions for ExampleProject' + assert_contain 'Latest Exceptions for PhantomProject' assert_contain 'No exceptions' assert_not_contain 'next page' assert_not_contain 'previous page' end - def test_projects_with_one_exception - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - + def test_projects_with_exceptions visit '/projects/ExampleProject' assert_contain 'Latest Exceptions for ExampleProject' assert_contain 'NameError in users#show' - assert_contain '# 2' + assert_contain 'NameError in users#save' + assert_contain 'NameError in users#delete' end def test_projects_pagination_latest 27.times do |i| - UberException.occurred(create_occurrence(action_name:"action_#{i}")) + UberException.occurred(create_occurrence(action_name:"action_#{i}", project_name: 'ThirdProject')) end - Exceptionist.esclient.refresh - visit '/projects/ExampleProject?sort_by=latest' + visit '/projects/ThirdProject?sort_by=latest' assert_contain 'next page' assert_not_contain 'previous page' @@ -133,12 +106,11 @@ def test_projects_pagination_latest def test_projects_pagination_frequent 27.times do |i| - UberException.occurred(create_occurrence(action_name:"action_#{i}")) + UberException.occurred(create_occurrence(action_name:"action_#{i}", project_name: 'ThirdProject')) end - Exceptionist.esclient.refresh - visit '/projects/ExampleProject?sort_by=frequent' + visit '/projects/ThirdProject?sort_by=frequent' assert_contain 'next page' assert_not_contain 'previous page' @@ -147,102 +119,58 @@ def test_projects_pagination_frequent assert_contain 'previous page' end - def test_projects_be_sorted_by_most_recent - UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-03-01')) - UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2009-02-01')) - - Exceptionist.esclient.refresh - - visit '/projects/ExampleProject' - - # TODO: how to def test order? - assert_contain 'NameError in users#index' - assert_contain 'NameError in users#show' - end - def test_projects_show_new_exceptions - UberException.occurred(create_occurrence(action_name:'show', occurred_at:'2010-07-01')) - UberException.occurred(create_occurrence(action_name:'index', occurred_at:'2010-08-01')) - - Exceptionist.esclient.refresh - - visit '/projects/ExampleProject/new_on/2010-07-01?mail_to=the@dude.org' + visit '/projects/ExampleProject/new_on/2011-01-01?mail_to=the@dude.org' assert_contain 'NameError in users#show' - assert_not_contain 'NameError in users#index' + assert_not_contain 'NameError in users#save' + assert_not_contain 'NameError in users#delete' end def test_projects_forget_old_exceptions - UberException.occurred(create_occurrence(action_name:'show', occurred_at:Time.now - (86400 * 50))) - UberException.occurred(create_occurrence(action_name:'index', occurred_at:Time.now)) - - Exceptionist.esclient.refresh - visit '/projects/ExampleProject/forget_exceptions', :post - assert_contain 'Deleted exceptions: 1' + assert_contain 'Deleted exceptions' end def test_projects_since_last_deploy_with_no_deploy assert_raises(ArgumentError) do - get '/projects/ExampleProject/since_last_deploy' + get '/projects/PhantomProject/since_last_deploy' end end - def test_projects_since_last_deploy_fresh_deploy - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - create_deploy - - Exceptionist.esclient.refresh - - visit '/projects/ExampleProject/since_last_deploy' + def test_projects_since_last_deploy_with_no_exceptions + visit '/projects/ThirdProject/since_last_deploy' assert_contain 'since last deploy' assert_contain 'No exceptions' end - def test_projects_since_last_deploy_old_deploy - create_deploy - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - + def test_projects_since_last_deploy visit '/projects/ExampleProject/since_last_deploy' assert_contain 'NameError in users#show' + assert_contain 'NameError in users#save' + assert_contain 'NameError in users#delete' assert_not_contain 'next page' assert_not_contain 'previous page' end def test_projects_since_last_deploy_ordered_by_occurrences_count - create_deploy - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - visit '/projects/ExampleProject/since_last_deploy?sort_by=frequent' assert_contain 'NameError in users#show' - assert_not_contain 'next page' - assert_not_contain 'previous page' + assert_contain 'NameError in users#save' + assert_contain 'NameError in users#delete' end def test_projects_since_last_deploy_pagination - create_deploy - 27.times do |i| - UberException.occurred(create_occurrence(action_name:"action_#{i}")) + UberException.occurred(create_occurrence(action_name:"action_#{i}", project_name: 'ThirdProject'),) end - Exceptionist.esclient.refresh - visit '/projects/ExampleProject/since_last_deploy' + visit '/projects/ThirdProject/since_last_deploy' assert_contain 'since last deploy' assert_contain 'next page' assert_not_contain 'previous page' @@ -254,119 +182,66 @@ def test_projects_since_last_deploy_pagination end def test_projects_new_exception_since_deploy - create_deploy - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - visit '/projects/ExampleProject' assert_contain 'new' - - create_deploy - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - - visit '/projects/ExampleProject' - assert_not_contain 'new' end - def test_projects_filter - UberException.occurred(create_occurrence) - - Exceptionist.esclient.refresh - + def test_projects_category_filter visit '/projects/ExampleProject' assert_have_selector '.no-category' - assert_have_no_selector '.low' - assert_have_no_selector '.high' + assert_have_selector '.high' + assert_have_selector '.low' visit '/projects/ExampleProject?category=no-category' assert_not_contain 'No exceptions' visit '/projects/ExampleProject?category=high' - assert_contain 'No exceptions' - end - - def test_projects_filter_change - @uber_exce = UberException.occurred(create_occurrence) - @uber_exce.update(category: 'high') - Exceptionist.esclient.refresh - - visit '/projects/ExampleProject' - assert_have_selector '.high' - assert_have_no_selector '.low' - assert_have_no_selector '.no-category' + assert_not_contain 'No exceptions' visit '/projects/ExampleProject?category=low' - assert_contain 'No exceptions' - - visit '/projects/ExampleProject?category=high' assert_not_contain 'No exceptions' end def test_projects_since_last_deploy_filter - create_deploy - @uber_exce = UberException.occurred(create_occurrence) - @uber_exce.update(category: 'high') - Exceptionist.esclient.refresh - visit '/projects/ExampleProject/since_last_deploy?' - assert_not_contain 'No exceptions' assert_have_selector '.high' - assert_have_no_selector '.low' - assert_have_no_selector '.no-category' + assert_have_selector '.low' + assert_have_selector '.no-category' - visit '/projects/ExampleProject/since_last_deploy?category=low' - assert_contain 'No exceptions' + visit '/projects/ExampleProject/since_last_deploy?category=no-category' + assert_have_selector '.no-category' + assert_have_no_selector '.high' visit '/projects/ExampleProject/since_last_deploy?category=high' - assert_not_contain 'No exceptions' + assert_have_selector '.high' + assert_have_no_selector '.low' + + visit '/projects/ExampleProject/since_last_deploy?category=low' + assert_have_selector '.low' + assert_have_no_selector '.no-category' end def test_projects_since_last_deploy_filter_sorted_by_occurrence - create_deploy - @uber_exce = UberException.occurred(create_occurrence) - @uber_exce.update(category: 'low') - Exceptionist.esclient.refresh - visit '/projects/ExampleProject/since_last_deploy?sort_by=frequent' - assert_not_contain 'No exceptions' + assert_have_selector '.no-category' + assert_have_selector '.high' assert_have_selector '.low' - assert_have_no_selector '.high' - assert_have_no_selector '.no-category' - visit '/projects/ExampleProject/since_last_deploy?category=low&sort_by=frequent' - assert_not_contain 'No exceptions' + visit '/projects/ExampleProject/since_last_deploy?category=no-category&sort_by=frequent' + assert_have_selector '.no-category' + assert_have_no_selector '.high' visit '/projects/ExampleProject/since_last_deploy?category=high&sort_by=frequent' - assert_contain 'No exceptions' - end - - def test_projects_sort_most_frequent_filter - @uber_exce = UberException.occurred(create_occurrence) - @uber_exce.update(category: 'low') - Exceptionist.esclient.refresh + assert_have_selector '.high' + assert_have_no_selector '.low' - visit '/projects/ExampleProject?sort_by=frequent' + visit '/projects/ExampleProject/since_last_deploy?category=low&sort_by=frequent' assert_have_selector '.low' - assert_have_no_selector '.high' assert_have_no_selector '.no-category' - - visit '/projects/ExampleProject?category=low&sort_by=frequent' - assert_not_contain 'No exceptions' - - visit '/projects/ExampleProject?category=high&sort_by=frequent' - assert_contain 'No exceptions' end def test_exceptions_show_a_minimal_occurrence - occurrence = create_occurrence - UberException.occurred(occurrence) - - Exceptionist.esclient.refresh - - visit "/exceptions/#{occurrence.uber_key}" + visit "/exceptions/#{@exce1.id}" assert_contain 'GET http://example.com' assert_contain 'NameError: undefined local variable or method dude' assert_contain 'Params:' @@ -376,16 +251,7 @@ def test_exceptions_show_a_minimal_occurrence end def test_exceptions_paginate_occurrences - occur1 = create_occurrence(url: 'http://example.com/?show=one') - occur2 = create_occurrence(url: 'http://example.com/?show=two') - occur3 = create_occurrence(url: 'http://example.com/?show=three') - UberException.occurred(occur1) - UberException.occurred(occur2) - UberException.occurred(occur3) - - Exceptionist.esclient.refresh - - visit "/exceptions/#{occur1.uber_key}" + visit "/exceptions/#{@exce3.id}" assert_contain 'Older' assert_contain 'Newer' assert_contain '3 of 3' @@ -404,40 +270,17 @@ def test_exceptions_paginate_occurrences end def test_projects_be_able_to_close_an_exception - UberException.occurred(create_occurrence(action_name:'show')) - UberException.occurred(create_occurrence(action_name:'index')) - - Exceptionist.esclient.refresh - visit '/projects/ExampleProject' - assert_contain 'NameError in users#show' - assert_contain 'NameError in users#index' click_link 'NameError in users#show' submit_form 'close' - Exceptionist.esclient.refresh - follow_redirect! # redirects back to project page assert_equal 'http://example.org/projects/ExampleProject', last_request.url assert_not_contain 'NameError in users#show' - assert_contain 'NameError in users#index' + assert_contain 'NameError in users#delete' end - - def test_deploys_no_deploys - visit '/deploys/ExampleProject' - assert_contain 'No deploys' - end - - def test_deploys_no_deploys - create_deploy - Exceptionist.esclient.refresh - - visit '/deploys/ExampleProject' - assert_not_contain 'No deploys' - end - end diff --git a/test/test_helper.rb b/test/test_helper.rb index 2eb7e18..107b687 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -29,7 +29,9 @@ # Configure Exceptionist.add_project 'ExampleProject', 'SECRET_API_KEY' -Exceptionist.add_project 'ExampleProject2', 'ANOTHER_SECRET_API_KEY' +Exceptionist.add_project 'OtherProject', 'ANOTHER_SECRET_OTHER_KEY' +Exceptionist.add_project 'ThirdProject', 'ANOTHER_SECRET_THIRD_KEY' +Exceptionist.add_project 'PhantomProject', 'ANOTHER_SECRET_PHANTOM_KEY' ## # Exceptionist specific helpers From ffbbd984bb56983ebc6339958dc0fe5ae146a703 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 11:05:20 +0200 Subject: [PATCH 197/293] enrich title of since last deploy page --- lib/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/app.rb b/lib/app.rb index 882a16d..52f9449 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -67,7 +67,7 @@ class ExceptionistApp < Sinatra::Base @category = { category: params[:category] } if params[:category] @uber_exceptions = UberException.find_since_last_deploy(project: @current_project.name, terms: [@category], from: @start) end - @title = "Exceptions since last deploy for #{@current_project.name}" + @title = "Exceptions since last deploy (#{format_time(@deploy.occurred_at)}) for project #{@current_project.name}" erb :index end From 02c2c83f03558e38d66755722a111cb59a795d13 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 11:08:38 +0200 Subject: [PATCH 198/293] move symbolize keys into helper module --- lib/es_client.rb | 6 +----- lib/helpers.rb | 3 +++ lib/models/deploy.rb | 6 +----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 32c2642..2794cd6 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -116,15 +116,11 @@ def create_deploy(attr) def transform(attr) attr.merge!(attr['_source']).delete('_source') - attr = symbolize_keys(attr) + attr = Helpers.symbolize_keys(attr) attr[:id] = attr.delete :_id attr end - def symbolize_keys(hash) - hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} - end - def create_search_query(terms, sort, from, size) sort = wrap_sort(sort) unless sort.empty? return { sort: sort, from: from, size: size } if terms.empty? diff --git a/lib/helpers.rb b/lib/helpers.rb index e4094ca..a8463db 100644 --- a/lib/helpers.rb +++ b/lib/helpers.rb @@ -5,4 +5,7 @@ def self.get_day_ago(days) today - (3600 * 24 * (days - 1)) # `days` days ago end + def self.symbolize_keys(hash) + hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} + end end diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 613df96..0d1ab6d 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -27,7 +27,7 @@ def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, si end def self.from_json(json) - attr = symbolize_keys(JSON.parse(json)) + attr = Helpers.symbolize_keys(JSON.parse(json)) attr['occurred_at'] = Time.now if attr['occurred_at'].nil? Deploy.new(attr) @@ -57,8 +57,4 @@ def to_hash occurred_at: occurred_at.is_a?(String) ? occurred_at : occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } end - - def self.symbolize_keys(hash) - hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} - end end From bab27801d4a7f4758a7282c304da3106da01503d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 11:52:19 +0200 Subject: [PATCH 199/293] add array wrap function to helper --- lib/helpers.rb | 5 +++++ test/helpers_test.rb | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/helpers.rb b/lib/helpers.rb index a8463db..6f32a0b 100644 --- a/lib/helpers.rb +++ b/lib/helpers.rb @@ -8,4 +8,9 @@ def self.get_day_ago(days) def self.symbolize_keys(hash) hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} end + + def self.wrap(args) + return [] unless args + args.is_a?(Array) ? args : [args] + end end diff --git a/test/helpers_test.rb b/test/helpers_test.rb index c2d144c..6e8d0aa 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -14,4 +14,12 @@ def test_get_day_ago assert_equal Time.local(2010, 5, 5), Helpers.get_day_ago(1) assert_equal Time.local(2010, 5, 1), Helpers.get_day_ago(5) end + + def test_wrap + assert_equal [1], Helpers.wrap(1) + assert_equal [], Helpers.wrap(nil) + assert_equal [1], Helpers.wrap([1]) + assert_equal [{test: 1}, {test: 2}], Helpers.wrap([{test: 1}, {test: 2}]) + assert_equal [{test: 1}], Helpers.wrap({test: 1}) + end end From 100ed72ae02443ed01e6613b93386255bef772c6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 11:54:35 +0200 Subject: [PATCH 200/293] refactor create search query --- lib/es_client.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 2794cd6..2ff4688 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -121,15 +121,16 @@ def transform(attr) attr end - def create_search_query(terms, sort, from, size) - sort = wrap_sort(sort) unless sort.empty? - return { sort: sort, from: from, size: size } if terms.empty? - { query: { filtered: { filter: { bool: { must: terms } } } }, sort: sort, from: from, size: size} + def create_search_query(filters, sort, from, size) + { query: wrap_filters(filters), sort: sort, from: from, size: size } end def wrap_sort(sort) - sort = [sort] if sort.class == Hash - sort.each { | field | add_ignore_unmapped(field) } + Helpers.wrap(sort).each { | field | add_ignore_unmapped(field) } + end + + def wrap_filters(filters) + { filtered: { filter: { bool: { must: filters } } } } end def add_ignore_unmapped(hash) From 7dffa2cdd909134bb26b5ee6eb351321d076d373 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 11:59:06 +0200 Subject: [PATCH 201/293] use keyword arguments for index function --- lib/es_client.rb | 2 +- lib/models/deploy.rb | 2 +- lib/models/occurrence.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 2ff4688..62c8cd4 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -53,7 +53,7 @@ def mget(ids: []) hash.docs.map { |doc| create_exception(doc) } end - def index(type, body) + def index(type: TYPE_OCCURRENCES, body: {}) response = @es.index(index: INDEX, type: type, body: body) Hashie::Mash.new(response) end diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 0d1ab6d..710b5ff 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -34,7 +34,7 @@ def self.from_json(json) end def save - deploy = Exceptionist.esclient.index('deploys', to_hash) + deploy = Exceptionist.esclient.index(type: 'deploys', body: to_hash) @id = deploy._id self end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 2616cd3..bfed3e3 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -112,7 +112,7 @@ def project # def save - occurrence = Exceptionist.esclient.index('occurrences', to_hash) + occurrence = Exceptionist.esclient.index(type: 'occurrences', body: to_hash) @id = occurrence._id self end From 98d4d5b66f795bf1879519ce9107fb5d35413182 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 12:59:39 +0200 Subject: [PATCH 202/293] use constant --- lib/es_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 62c8cd4..d4ff919 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -62,7 +62,7 @@ def update(type, id, body) @es.update(index: INDEX, type: type, id: id, body: body) end - def count(type: 'occurrences', filters: []) + def count(type: TYPE_OCCURRENCES, filters: []) filters = [filters] if filters.class == Hash query = { query: { filtered: { filter: { bool: { must: filters } } } } } From 92ad72137a94cf4d9c84cb8d767806ac16b38b9e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:12:52 +0200 Subject: [PATCH 203/293] reuse wrap filters function --- lib/es_client.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index d4ff919..c42b6ce 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -35,7 +35,7 @@ def search_deploys(filters: {}, sort: {}, from: 0, size: 25) def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE - query = { query: { filtered: { filter: { bool: { must: filters } } } }, aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } + query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query, search_type: 'count') hash = Hashie::Mash.new(response) hash.aggregations.exceptions.buckets @@ -63,10 +63,7 @@ def update(type, id, body) end def count(type: TYPE_OCCURRENCES, filters: []) - filters = [filters] if filters.class == Hash - query = { query: { filtered: { filter: { bool: { must: filters } } } } } - - @es.count(index: INDEX, type: type, body: query)['count'] + @es.count(index: INDEX, type: type, body: { query: wrap_filters(filters) } )['count'] end def delete_by_query(query: { match_all: {} }) @@ -130,7 +127,7 @@ def wrap_sort(sort) end def wrap_filters(filters) - { filtered: { filter: { bool: { must: filters } } } } + { filtered: { filter: { bool: { must: Helpers.wrap(filters) } } } } end def add_ignore_unmapped(hash) From 02fb97f9463496f6ce7cd03ae0ef6de0fa535826 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:15:43 +0200 Subject: [PATCH 204/293] use keyword arguments for update function --- lib/es_client.rb | 2 +- lib/models/uber_exception.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index c42b6ce..9829e59 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -58,7 +58,7 @@ def index(type: TYPE_OCCURRENCES, body: {}) Hashie::Mash.new(response) end - def update(type, id, body) + def update(type: TYPE_EXCEPTIONS, id: 0, body: {}) @es.update(index: INDEX, type: type, id: id, body: body) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index d459f4f..c53dc63 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -104,7 +104,7 @@ def self.occurred(occurrence) first_timestamp = first_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%L%z") hash = occurrence.to_hash hash[:id] = occurrence.id - Exceptionist.esclient.update('exceptions', occurrence.uber_key, { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', + Exceptionist.esclient.update(id: occurrence.uber_key, body: { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1, category: 'no-category'}, params: { occurrence: hash, timestamp: first_timestamp} }) Exceptionist.esclient.get_exception(occurrence.uber_key) @@ -131,11 +131,11 @@ def forget! end def close! - Exceptionist.esclient.update('exceptions', @id, { doc: { closed: true } }) + Exceptionist.esclient.update(id: @id, body: { doc: { closed: true } }) end def update(doc) - Exceptionist.esclient.update('exceptions', @id, { doc: doc }) + Exceptionist.esclient.update(id: @id, body: { doc: doc }) end def current_occurrence(position) From 9b72d47a389236ccf43f5b1a712c476388f950cf Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:20:08 +0200 Subject: [PATCH 205/293] use keyword arguments for search ids --- lib/es_client.rb | 2 +- lib/models/uber_exception.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 9829e59..d503eda 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -41,7 +41,7 @@ def search_aggs(filters, aggs) hash.aggregations.exceptions.buckets end - def search_ids(type, ids) + def search_ids(type: TYPE_EXCEPTIONS, ids: []) response = @es.search(index: INDEX, type: type, body: { query: { ids: { values: ids } } } ) hash = Hashie::Mash.new(response) hash.hits.hits.map { |doc| create_exception(doc) } diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index c53dc63..efbab7c 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -84,7 +84,7 @@ def self.find_new_on(project, day) next_day = day + 86400 buckets = Exceptionist.esclient.search_aggs({ term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, 'uber_key' ) - uber_exceptions = Exceptionist.esclient.search_ids('exceptions', buckets.map { |occ| occ['key'] } ) + uber_exceptions = Exceptionist.esclient.search_ids(ids: buckets.map { |occ| occ['key'] } ) uber_exceptions.select { |uber_exp| uber_exp.first_occurred_at >= day && uber_exp.first_occurred_at < next_day } end From 4da2de711d719923b8e231ca2c920392a5f340db Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:21:01 +0200 Subject: [PATCH 206/293] refactor search deploy --- lib/es_client.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index d503eda..35b0846 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -27,9 +27,7 @@ def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) end def search_deploys(filters: {}, sort: {}, from: 0, size: 25) - query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: TYPE_DEPLOYS, body: query) - hash = Hashie::Mash.new(response) + hash = search(type: TYPE_DEPLOYS, filters: filters, sort: sort, from: from, size: size) hash.hits.hits.map { |doc| create_deploy(doc) } end From 6aab682dc19e2c83a2fa371baf8a7e8a5bd58ce6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:21:53 +0200 Subject: [PATCH 207/293] rearrange functions --- lib/es_client.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 35b0846..b485552 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -10,9 +10,9 @@ def initialize(endpoint) @es = Elasticsearch::Client.new(host: endpoint) end - def search_occurrences(filters: {}, sort: {}, from: 0, size: 25) - hash = search(filters: filters, sort: sort, from: from, size: size) - hash.hits.hits.map { |doc| create_occurrence(doc) } + def search_deploys(filters: {}, sort: {}, from: 0, size: 25) + hash = search(type: TYPE_DEPLOYS, filters: filters, sort: sort, from: from, size: size) + hash.hits.hits.map { |doc| create_deploy(doc) } end def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) @@ -20,17 +20,17 @@ def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) hash.hits.hits.map { |doc| create_exception(doc) } end + def search_occurrences(filters: {}, sort: {}, from: 0, size: 25) + hash = search(filters: filters, sort: sort, from: from, size: size) + hash.hits.hits.map { |doc| create_occurrence(doc) } + end + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: type, body: query) Hashie::Mash.new(response) end - def search_deploys(filters: {}, sort: {}, from: 0, size: 25) - hash = search(type: TYPE_DEPLOYS, filters: filters, sort: sort, from: from, size: size) - hash.hits.hits.map { |doc| create_deploy(doc) } - end - def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } From 832aba27f4d2cc72110fceb1e998eab19d20f4fc Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:25:51 +0200 Subject: [PATCH 208/293] make search funtion private --- lib/es_client.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index b485552..aed08b6 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -25,12 +25,6 @@ def search_occurrences(filters: {}, sort: {}, from: 0, size: 25) hash.hits.hits.map { |doc| create_occurrence(doc) } end - def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) - query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: type, body: query) - Hashie::Mash.new(response) - end - def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } @@ -94,6 +88,12 @@ def refresh end private + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: type, body: query) + Hashie::Mash.new(response) + end + def create_occurrence(attr) attr = transform(attr) Occurrence.new(attr) From 4f5e8a31c8b177705883b48b67f413cdaee1f296 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:26:40 +0200 Subject: [PATCH 209/293] use keyword arguments for delete function --- lib/es_client.rb | 2 +- lib/models/uber_exception.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index aed08b6..098f8c7 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -50,7 +50,7 @@ def index(type: TYPE_OCCURRENCES, body: {}) Hashie::Mash.new(response) end - def update(type: TYPE_EXCEPTIONS, id: 0, body: {}) + def update(type: TYPE_EXCEPTIONS, id: -1, body: {}) @es.update(index: INDEX, type: type, id: id, body: body) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index efbab7c..bbf36ea 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -127,7 +127,7 @@ def self.forget_old_exceptions(project, days=0) def forget! Occurrence.delete_all_for(id) - Exceptionist.esclient.delete('exceptions', id) + Exceptionist.esclient.delete(id: id) end def close! From f739055c4bf1043542a56f3b2790cfdf6aee7b0d Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:27:28 +0200 Subject: [PATCH 210/293] set id default to -1, no default value ruby 2.1 --- lib/es_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 098f8c7..1aa5322 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -62,7 +62,7 @@ def delete_by_query(query: { match_all: {} }) @es.delete_by_query( index: INDEX, body: { query: query } ) end - def delete(type, id) + def delete(type: TYPE_EXCEPTIONS, id: -1) @es.delete(index: INDEX, type: type, id: id) end From 1c5bd59f1567ce8e6e364ca90b007bd45790e9e3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:28:54 +0200 Subject: [PATCH 211/293] add blank lines for readability --- lib/es_client.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/es_client.rb b/lib/es_client.rb index 1aa5322..89f6265 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -1,5 +1,7 @@ class ESClient + attr_accessor :es, :host, :port + INDEX = 'exceptionist' TYPE_EXCEPTIONS = 'exceptions' TYPE_OCCURRENCES = 'occurrences' From 0e55f7433480ae248dd73c57c948c27c362b1bb4 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:36:41 +0200 Subject: [PATCH 212/293] use helper function to wrap --- lib/models/occurrence.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index bfed3e3..b3c469b 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -47,7 +47,7 @@ def self.find_next(uber_key, date) def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) raise ArgumentError, 'position has to be >= 0' if from < 0 - filters = [filters] if filters.class == Hash + filters = Helpers.wrap(filters) filters << { term: { uber_key: uber_key } } unless uber_key.empty? Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) end @@ -61,7 +61,7 @@ def self.count_since(uber_key, time) end def self.count(project: '', filters: {}) - filters = [filters] if filters.class == Hash + filters = Helpers.wrap(filters) filters << { term: { project_name: project } } unless project.empty? Exceptionist.esclient.count( filters: filters ) end From 6404a2507cd07b935e13c209c3165c86574c3141 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:37:19 +0200 Subject: [PATCH 213/293] initialisation ensures that time is in the field --- lib/models/occurrence.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index b3c469b..10e3ae5 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -99,10 +99,6 @@ def user_agent cgi_data ? cgi_data['HTTP_USER_AGENT'] : nil end - def occurred_at - @occurred_at.is_a?(String) ? Time.parse(@occurred_at) : @occurred_at - end - def project Project.new(project_name) end From 9239a8a4e52e6d015f78d37690244995a5e029ed Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:43:57 +0200 Subject: [PATCH 214/293] create helper for ES day and time format --- lib/helpers.rb | 8 ++++++++ lib/models/occurrence.rb | 14 +++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/helpers.rb b/lib/helpers.rb index 6f32a0b..448c028 100644 --- a/lib/helpers.rb +++ b/lib/helpers.rb @@ -13,4 +13,12 @@ def self.wrap(args) return [] unless args args.is_a?(Array) ? args : [args] end + + def self.es_time(date) + date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + end + + def self.es_day(date) + date.strftime('%Y-%m-%d') + end end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 10e3ae5..9fb084a 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -33,7 +33,7 @@ def self.find_last_for(uber_key) end def self.find_since(uber_key: "", date: Time.local, from: 0, size: 25) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, from: from, size: size) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, from: from, size: size) end def self.find_by_name(project, size=25) @@ -41,7 +41,7 @@ def self.find_by_name(project, size=25) end def self.find_next(uber_key, date) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first end def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) @@ -53,11 +53,11 @@ def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } end def self.count_all_on(project, day) - Occurrence.count( project: project, filters: { term: { occurred_at_day: day.strftime('%Y-%m-%d') } }) + Occurrence.count( project: project, filters: { term: { occurred_at_day: Helpers.es_day(day) } }) end - def self.count_since(uber_key, time) - Occurrence.count(filters: [{ range: { occurred_at: { gte: time.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } }, { term: { uber_key: uber_key } }] ) + def self.count_since(uber_key, date) + Occurrence.count(filters: [{ range: { occurred_at: { gte: Helpers.es_time(date) } } }, { term: { uber_key: uber_key } }] ) end def self.count(project: '', filters: {}) @@ -120,8 +120,8 @@ def to_hash parameters: parameters, cgi_data: cgi_data, url: url, - occurred_at: occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z"), - occurred_at_day: occurred_at.strftime('%Y-%m-%d'), + occurred_at: Helpers.es_time(occurred_at), + occurred_at_day: Helpers.es_day(occurred_at), exception_backtrace: exception_backtrace, controller_name: controller_name, environment: environment, From f39bdd8eedeadc03c7f5f1d38e8365a2f39e0739 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:46:12 +0200 Subject: [PATCH 215/293] move raise exce to root function --- lib/es_client.rb | 2 ++ lib/models/occurrence.rb | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 89f6265..a77cd46 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -91,6 +91,8 @@ def refresh private def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) + raise ArgumentError, 'from has to be >= 0' if from < 0 + query = create_search_query(filters, sort, from, size) response = @es.search(index: INDEX, type: type, body: query) Hashie::Mash.new(response) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 9fb084a..99e1f09 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -45,8 +45,6 @@ def self.find_next(uber_key, date) end def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) - raise ArgumentError, 'position has to be >= 0' if from < 0 - filters = Helpers.wrap(filters) filters << { term: { uber_key: uber_key } } unless uber_key.empty? Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) From fb88e932356bf42edde6898670b115806f406ef6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:53:16 +0200 Subject: [PATCH 216/293] user generic function instead of find_by_name --- lib/app.rb | 2 +- lib/models/occurrence.rb | 4 ---- test/occurrence_test.rb | 10 ++-------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 52f9449..9c9092f 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -73,7 +73,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/river' do @current_project = Project.new(params[:project]) - @occurrences = Occurrence.find_by_name(@current_project.name) + @occurrences = Occurrence.find( filters: { term: { project_name: @current_project.name } } ) @title = "Latest Occurrences for #{@current_project.name}" erb :river diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 99e1f09..9a3ac68 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -36,10 +36,6 @@ def self.find_since(uber_key: "", date: Time.local, from: 0, size: 25) Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, from: from, size: size) end - def self.find_by_name(project, size=25) - Occurrence.find(filters: { term: { project_name: project } }, size: size) - end - def self.find_next(uber_key, date) Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first end diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index 41e844d..beec499 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -40,11 +40,10 @@ def test_title_accessor def test_delete_all_for Occurrence.delete_all_for(@occur11.uber_key) - Exceptionist.esclient.refresh - assert_equal 5, Occurrence.find_by_name('ExampleProject').size - assert_equal 1, Occurrence.find_by_name('OtherProject').size + assert_equal 5, Occurrence.find(filters: { term: { project_name: 'ExampleProject' } } ).size + assert_equal 1, Occurrence.find(filters: { term: { project_name: 'OtherProject' } } ).size end def test_find_first_for @@ -59,11 +58,6 @@ def test_find_since assert_equal [@occur19, @occur18], Occurrence.find_since(uber_key: @occur18.uber_key, date: Time.local(2011, 1, 7, 12, 0)) end - def test_find_by_name - assert_equal 14, Occurrence.find_by_name('ExampleProject').size - assert_equal [@occur41], Occurrence.find_by_name('OtherProject') - end - def test_find_next assert_equal @occur19, Occurrence.find_next(@occur18.uber_key, Time.local(2011, 1, 8, 12, 0)) assert_equal @occur12, Occurrence.find_next(@occur11.uber_key, Time.local(2011, 1, 1, 12, 0)) From 88b236521594f8d6b32491eec616433a7695ee44 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 13:56:53 +0200 Subject: [PATCH 217/293] rearrange functions --- lib/models/occurrence.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 9a3ac68..4ad45ea 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -60,14 +60,6 @@ def self.count(project: '', filters: {}) Exceptionist.esclient.count( filters: filters ) end - def ==(other) - id == other.id - end - - def inspect - "(Occurrence id=#{id} uber_key=#{uber_key})" - end - # # accessors # @@ -186,6 +178,14 @@ def self.parse_optional_element(doc, xpath) element ? element.content : nil end + def ==(other) + id == other.id + end + + def inspect + "(Occurrence id=#{id} uber_key=#{uber_key})" + end + private def generate_uber_key From 767317ed8d0d05e77b53a4e258836adf35a9ad1c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:07:47 +0200 Subject: [PATCH 218/293] use keyword arguments for get function --- lib/es_client.rb | 4 ++-- lib/models/uber_exception.rb | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index a77cd46..2944ddf 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -80,8 +80,8 @@ def get_mapping(type) @es.indices.get_mapping(index: INDEX, type: type) end - def get_exception(id) - response = @es.get(index: INDEX, type: TYPE_EXCEPTIONS, id: id) + def get(type: '', id: id) + response = @es.get(index: INDEX, type: type, id: id) create_exception(response) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index bbf36ea..66f0564 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,6 +1,8 @@ class UberException attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at, :category + TYPE_EXCEPTIONS = 'exceptions' + def initialize(attributes) attributes.each do |key, value| instance_variable_set("@#{key}", value) @@ -19,7 +21,7 @@ def self.count_since(project: '', date: '') end def self.get(uber_key) - Exceptionist.esclient.get_exception(uber_key) + Exceptionist.esclient.get(type: TYPE_EXCEPTIONS, id: uber_key) end def self.find_sorted_by_occurrences_count(terms: [], from: 0, size: 25) @@ -94,7 +96,7 @@ def self.occurred(occurrence) #TODO maybe remove when events arrive sorted begin - exec = Exceptionist.esclient.get_exception(occurrence.uber_key) + exec = UberException.get(occurrence.uber_key) occurrence = exec.last_occurrence.occurred_at < first_timestamp ? occurrence : exec.last_occurrence first_timestamp = first_timestamp < exec.first_occurred_at ? first_timestamp : exec.first_occurred_at rescue Elasticsearch::Transport::Transport::Errors::NotFound @@ -107,7 +109,7 @@ def self.occurred(occurrence) Exceptionist.esclient.update(id: occurrence.uber_key, body: { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1, category: 'no-category'}, params: { occurrence: hash, timestamp: first_timestamp} }) - Exceptionist.esclient.get_exception(occurrence.uber_key) + UberException.get(occurrence.uber_key) end def self.forget_old_exceptions(project, days=0) From e7f077ab3b2b1fa2e56b0b46a43234e6ab53cee1 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:09:19 +0200 Subject: [PATCH 219/293] use constant --- lib/models/uber_exception.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 66f0564..4e0ec84 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -13,11 +13,11 @@ def initialize(attributes) end def self.count_all(project) - Exceptionist.esclient.count(type: 'exceptions', filters: { term: { project_name: project } } ) + Exceptionist.esclient.count(type: TYPE_EXCEPTIONS, filters: { term: { project_name: project } } ) end def self.count_since(project: '', date: '') - Exceptionist.esclient.count(type: 'exceptions', filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + Exceptionist.esclient.count(type: TYPE_EXCEPTIONS, filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) end def self.get(uber_key) @@ -33,7 +33,7 @@ def self.find_since_last_deploy(project: '', terms: [], from: 0, size: 25) ids = [] agg_exces.each { |occurr| ids << occurr['key'] } - exces = find( terms: terms.compact << { closed: false }, filters: [ { ids: { type: 'exceptions', values: ids } } ], from: from, size: size ) + exces = find( terms: terms.compact << { closed: false }, filters: [ { ids: { type: TYPE_EXCEPTIONS, values: ids } } ], from: from, size: size ) exces.each do |exce| agg_exces.each do |occurr| if occurr['key'] == exce.id From c81a4d7dc2a5518995c40bc3e9c53f197619d0d9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:12:04 +0200 Subject: [PATCH 220/293] remove double check --- lib/models/deploy.rb | 1 - lib/models/uber_exception.rb | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 710b5ff..26c62c1 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -22,7 +22,6 @@ def self.find_last_deploy(project) end def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) - raise ArgumentError, 'position has to be >= 0' if from < 0 Exceptionist.esclient.search_deploys( filters: filters, sort: sort, from: from, size: size ) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 4e0ec84..5a1cc97 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -75,8 +75,6 @@ def self.aggregation_since_last_deploy(project) end def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) - raise ArgumentError, 'position has to be >= 0' if from < 0 - terms = terms.map { |term| { term: term } unless term.nil? } terms << { term: { closed: false } } Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) From 2bfd4f2f8019050f26aeea1172381fcadb12e8f9 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:22:25 +0200 Subject: [PATCH 221/293] improve find since last deploy function --- lib/models/uber_exception.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 5a1cc97..1d979e3 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -52,9 +52,8 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', catego ids = [] agg_exces.each { |occurr| ids << occurr['key'] } exces = Exceptionist.esclient.mget(ids: ids) - exces = exces.select{ |exce| exce.category == category } unless category.nil? - exces = exces.slice(from, size) - exces.each do |exce| + exces.select!{ |exce| exce.category == category && !exce.closed } unless category.nil? + exces.slice(from, size).each do |exce| agg_exces.each do |occurr| if occurr['key'] == exce.id exce.occurrences_count = occurr['doc_count'] From a337226d8105173b045a73fd7d5bdb2a9b2ae5c6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:23:09 +0200 Subject: [PATCH 222/293] remove unused variable --- lib/app.rb | 2 +- lib/models/uber_exception.rb | 2 +- test/uber_exception_test.rb | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 9c9092f..90c908f 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -82,7 +82,7 @@ class ExceptionistApp < Sinatra::Base get '/projects/:project/new_on/:day' do @day = Time.parse(params[:day]) @current_project = Project.new(params[:project]) - @uber_exceptions = UberException.find_new_on(@current_project.name, @day) + @uber_exceptions = UberException.find_new_on(@day) message_body = erb(:new_exceptions, :layout => false) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 1d979e3..5246df8 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -79,7 +79,7 @@ def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) end - def self.find_new_on(project, day) + def self.find_new_on(day) next_day = day + 86400 buckets = Exceptionist.esclient.search_aggs({ term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, 'uber_key' ) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 827557f..87fafbd 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -113,10 +113,10 @@ def test_find_since_last_deploy_ordered_by_occurrences_count end def test_find_new_on - exceptions = UberException.find_new_on('ExampleProject', Time.local(2011, 1, 1)) + exceptions = UberException.find_new_on(Time.local(2011, 1, 1)) assert_equal 1, exceptions.size - assert_equal [], UberException.find_new_on('OtherProject', Time.local(2011, 1, 4, 12, 30)) + assert_equal [], UberException.find_new_on(Time.local(2011, 1, 4, 12, 30)) end def test_forget_old_exceptions From 50d8e975365d46a8097d5bc213652f2cca9b2005 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:27:59 +0200 Subject: [PATCH 223/293] remove new on day feature --- lib/app.rb | 14 -------------- lib/models/uber_exception.rb | 9 --------- test/integration_test.rb | 8 -------- test/uber_exception_test.rb | 7 ------- views/new_exceptions.erb | 24 ------------------------ 5 files changed, 62 deletions(-) delete mode 100644 views/new_exceptions.erb diff --git a/lib/app.rb b/lib/app.rb index 90c908f..57af37d 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -79,20 +79,6 @@ class ExceptionistApp < Sinatra::Base erb :river end - get '/projects/:project/new_on/:day' do - @day = Time.parse(params[:day]) - @current_project = Project.new(params[:project]) - @uber_exceptions = UberException.find_new_on(@day) - - message_body = erb(:new_exceptions, :layout => false) - - if params[:mail_to] - Mailer.deliver_new_exceptions(@current_project, @day, params[:mail_to], message_body) - end - - message_body - end - post '/projects/:project/forget_exceptions' do days = params[:days] ? params[:days].to_i : 31 deleted = UberException.forget_old_exceptions(params[:project], days) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 5246df8..ed19dae 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -79,15 +79,6 @@ def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) end - def self.find_new_on(day) - next_day = day + 86400 - - buckets = Exceptionist.esclient.search_aggs({ term: { occurred_at_day: day.strftime('%Y-%m-%d') } }, 'uber_key' ) - uber_exceptions = Exceptionist.esclient.search_ids(ids: buckets.map { |occ| occ['key'] } ) - - uber_exceptions.select { |uber_exp| uber_exp.first_occurred_at >= day && uber_exp.first_occurred_at < next_day } - end - def self.occurred(occurrence) first_timestamp = occurrence.occurred_at diff --git a/test/integration_test.rb b/test/integration_test.rb index c63164c..fd3a2a1 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -119,14 +119,6 @@ def test_projects_pagination_frequent assert_contain 'previous page' end - def test_projects_show_new_exceptions - visit '/projects/ExampleProject/new_on/2011-01-01?mail_to=the@dude.org' - - assert_contain 'NameError in users#show' - assert_not_contain 'NameError in users#save' - assert_not_contain 'NameError in users#delete' - end - def test_projects_forget_old_exceptions visit '/projects/ExampleProject/forget_exceptions', :post diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index 87fafbd..e53ede9 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -112,13 +112,6 @@ def test_find_since_last_deploy_ordered_by_occurrences_count assert_equal Time.local(2011, 1, 2), uber_exces[2].first_occurred_at end - def test_find_new_on - exceptions = UberException.find_new_on(Time.local(2011, 1, 1)) - assert_equal 1, exceptions.size - - assert_equal [], UberException.find_new_on(Time.local(2011, 1, 4, 12, 30)) - end - def test_forget_old_exceptions days = (Time.now - Time.local(2011, 1, 1)) / 86400 # shouldn't forget anything diff --git a/views/new_exceptions.erb b/views/new_exceptions.erb deleted file mode 100644 index 89d7bfd..0000000 --- a/views/new_exceptions.erb +++ /dev/null @@ -1,24 +0,0 @@ - - -
-

Total exceptions: <%= @current_project.total_count_on(@day) %>

- -

There have been <%= @uber_exceptions.size %> new exceptions:

- - -
From 0ea5f56c264b8b47fc7ef24308a6a4e049cf9bf8 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:31:51 +0200 Subject: [PATCH 224/293] add blank line --- lib/models/uber_exception.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index ed19dae..c773c35 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -1,4 +1,5 @@ class UberException + attr_accessor :id, :project_name, :occurrences_count, :closed, :last_occurrence, :first_occurred_at, :category TYPE_EXCEPTIONS = 'exceptions' From d62041cb8463fabc08112b970d39d6b32351d232 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:34:39 +0200 Subject: [PATCH 225/293] move function into helper class --- lib/helpers.rb | 12 ++++++++++++ lib/models/project.rb | 14 +------------- lib/models/uber_exception.rb | 2 +- test/helpers_test.rb | 5 +++++ test/project_test.rb | 5 ----- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/helpers.rb b/lib/helpers.rb index 448c028..3f64e0a 100644 --- a/lib/helpers.rb +++ b/lib/helpers.rb @@ -5,6 +5,18 @@ def self.get_day_ago(days) today - (3600 * 24 * (days - 1)) # `days` days ago end + def self.last_n_days(days) + start = Helpers.get_day_ago(days) + today = Time.now + + n_days = [] + begin + n_days << Time.utc(start.year, start.month, start.day) + end while (start += 86400) <= today + + n_days + end + def self.symbolize_keys(hash) hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} end diff --git a/lib/models/project.rb b/lib/models/project.rb index 7289b3d..7d57b15 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -10,19 +10,7 @@ def exceptions_count end def last_thirty_days - Project.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } - end - - def self.last_n_days(days) - start = Helpers.get_day_ago(days) - today = Time.now - - n_days = [] - begin - n_days << Time.utc(start.year, start.month, start.day) - end while (start += 86400) <= today - - n_days + Helpers.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } end def last_deploy diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index c773c35..1c65af5 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -156,7 +156,7 @@ def occurrences_count_on(date) end def last_thirty_days - Project.last_n_days(30).map { |day| [day, occurrences_count_on(day)] } + Helpers.last_n_days(30).map { |day| [day, occurrences_count_on(day)] } end def ==(other) diff --git a/test/helpers_test.rb b/test/helpers_test.rb index 6e8d0aa..1891c4d 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -15,6 +15,11 @@ def test_get_day_ago assert_equal Time.local(2010, 5, 1), Helpers.get_day_ago(5) end + def test_last_n_days + assert_equal 2, Helpers.last_n_days(2).size + assert_equal 4, Helpers.last_n_days(4).size + end + def test_wrap assert_equal [1], Helpers.wrap(1) assert_equal [], Helpers.wrap(nil) diff --git a/test/project_test.rb b/test/project_test.rb index bf48747..8cc734f 100644 --- a/test/project_test.rb +++ b/test/project_test.rb @@ -6,11 +6,6 @@ def setup clear_collections end - def test_last_n_days - assert_equal 2, Project.last_n_days(2).size - assert_equal 4, Project.last_n_days(4).size - end - def test_exceptions_count project = Project.new("ExampleProject") From c7b57d6df6891430c10e6f9716207c127426af09 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:37:14 +0200 Subject: [PATCH 226/293] rename helpers to helper --- lib/boot.rb | 2 +- lib/es_client.rb | 6 +++--- lib/{helpers.rb => helper.rb} | 4 ++-- lib/models/deploy.rb | 2 +- lib/models/occurrence.rb | 16 ++++++++-------- lib/models/project.rb | 4 ++-- lib/models/uber_exception.rb | 2 +- test/helpers_test.rb | 18 +++++++++--------- 8 files changed, 27 insertions(+), 27 deletions(-) rename lib/{helpers.rb => helper.rb} (92%) diff --git a/lib/boot.rb b/lib/boot.rb index 8a6b2d7..fa5c059 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -20,4 +20,4 @@ require 'mapping/mapping_helper' require 'exceptionist' -require 'helpers' +require 'helper' diff --git a/lib/es_client.rb b/lib/es_client.rb index 2944ddf..f236afb 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -115,7 +115,7 @@ def create_deploy(attr) def transform(attr) attr.merge!(attr['_source']).delete('_source') - attr = Helpers.symbolize_keys(attr) + attr = Helper.symbolize_keys(attr) attr[:id] = attr.delete :_id attr end @@ -125,11 +125,11 @@ def create_search_query(filters, sort, from, size) end def wrap_sort(sort) - Helpers.wrap(sort).each { | field | add_ignore_unmapped(field) } + Helper.wrap(sort).each { | field | add_ignore_unmapped(field) } end def wrap_filters(filters) - { filtered: { filter: { bool: { must: Helpers.wrap(filters) } } } } + { filtered: { filter: { bool: { must: Helper.wrap(filters) } } } } end def add_ignore_unmapped(hash) diff --git a/lib/helpers.rb b/lib/helper.rb similarity index 92% rename from lib/helpers.rb rename to lib/helper.rb index 3f64e0a..345e286 100644 --- a/lib/helpers.rb +++ b/lib/helper.rb @@ -1,4 +1,4 @@ -module Helpers +module Helper def self.get_day_ago(days) today = Time.now @@ -6,7 +6,7 @@ def self.get_day_ago(days) end def self.last_n_days(days) - start = Helpers.get_day_ago(days) + start = Helper.get_day_ago(days) today = Time.now n_days = [] diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 26c62c1..f88a603 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -26,7 +26,7 @@ def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, si end def self.from_json(json) - attr = Helpers.symbolize_keys(JSON.parse(json)) + attr = Helper.symbolize_keys(JSON.parse(json)) attr['occurred_at'] = Time.now if attr['occurred_at'].nil? Deploy.new(attr) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 4ad45ea..87ae5af 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -33,29 +33,29 @@ def self.find_last_for(uber_key) end def self.find_since(uber_key: "", date: Time.local, from: 0, size: 25) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, from: from, size: size) + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helper.es_time(date) } } }, from: from, size: size) end def self.find_next(uber_key, date) - Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helpers.es_time(date) } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first + Occurrence.find(uber_key: uber_key, filters: { range: { occurred_at: { gte: Helper.es_time(date) } } }, sort: { occurred_at: { order: 'asc' } }, size: 1).first end def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) - filters = Helpers.wrap(filters) + filters = Helper.wrap(filters) filters << { term: { uber_key: uber_key } } unless uber_key.empty? Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) end def self.count_all_on(project, day) - Occurrence.count( project: project, filters: { term: { occurred_at_day: Helpers.es_day(day) } }) + Occurrence.count( project: project, filters: { term: { occurred_at_day: Helper.es_day(day) } }) end def self.count_since(uber_key, date) - Occurrence.count(filters: [{ range: { occurred_at: { gte: Helpers.es_time(date) } } }, { term: { uber_key: uber_key } }] ) + Occurrence.count(filters: [{ range: { occurred_at: { gte: Helper.es_time(date) } } }, { term: { uber_key: uber_key } }] ) end def self.count(project: '', filters: {}) - filters = Helpers.wrap(filters) + filters = Helper.wrap(filters) filters << { term: { project_name: project } } unless project.empty? Exceptionist.esclient.count( filters: filters ) end @@ -106,8 +106,8 @@ def to_hash parameters: parameters, cgi_data: cgi_data, url: url, - occurred_at: Helpers.es_time(occurred_at), - occurred_at_day: Helpers.es_day(occurred_at), + occurred_at: Helper.es_time(occurred_at), + occurred_at_day: Helper.es_day(occurred_at), exception_backtrace: exception_backtrace, controller_name: controller_name, environment: environment, diff --git a/lib/models/project.rb b/lib/models/project.rb index 7d57b15..1cac21e 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -10,7 +10,7 @@ def exceptions_count end def last_thirty_days - Helpers.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } + Helper.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } end def last_deploy @@ -18,7 +18,7 @@ def last_deploy end def deploys_last_thirty_days - since = Helpers.get_day_ago(30) + since = Helper.get_day_ago(30) Deploy.find_by_project_since(@name, since) end diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 1c65af5..8c3fc29 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -156,7 +156,7 @@ def occurrences_count_on(date) end def last_thirty_days - Helpers.last_n_days(30).map { |day| [day, occurrences_count_on(day)] } + Helper.last_n_days(30).map { |day| [day, occurrences_count_on(day)] } end def ==(other) diff --git a/test/helpers_test.rb b/test/helpers_test.rb index 1891c4d..4c9dac1 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -11,20 +11,20 @@ def teardown end def test_get_day_ago - assert_equal Time.local(2010, 5, 5), Helpers.get_day_ago(1) - assert_equal Time.local(2010, 5, 1), Helpers.get_day_ago(5) + assert_equal Time.local(2010, 5, 5), Helper.get_day_ago(1) + assert_equal Time.local(2010, 5, 1), Helper.get_day_ago(5) end def test_last_n_days - assert_equal 2, Helpers.last_n_days(2).size - assert_equal 4, Helpers.last_n_days(4).size + assert_equal 2, Helper.last_n_days(2).size + assert_equal 4, Helper.last_n_days(4).size end def test_wrap - assert_equal [1], Helpers.wrap(1) - assert_equal [], Helpers.wrap(nil) - assert_equal [1], Helpers.wrap([1]) - assert_equal [{test: 1}, {test: 2}], Helpers.wrap([{test: 1}, {test: 2}]) - assert_equal [{test: 1}], Helpers.wrap({test: 1}) + assert_equal [1], Helper.wrap(1) + assert_equal [], Helper.wrap(nil) + assert_equal [1], Helper.wrap([1]) + assert_equal [{test: 1}, {test: 2}], Helper.wrap([{test: 1}, {test: 2}]) + assert_equal [{test: 1}], Helper.wrap({test: 1}) end end From a825c3edabf6027d09417adba46987fbfc7bbe26 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:38:30 +0200 Subject: [PATCH 227/293] rearrange functions --- lib/models/project.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/models/project.rb b/lib/models/project.rb index 1cac21e..3994532 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -26,14 +26,6 @@ def total_count_on(day) Occurrence.count_all_on(name, day) end - def ==(other) - name == other.name - end - - def inspect - "(Project name=#{name})" - end - def self.find_by_key(api_key) project = Exceptionist.projects.find { |name, project_key| project_key == api_key } project ? Project.new(project.first) : nil @@ -42,4 +34,12 @@ def self.find_by_key(api_key) def self.all Exceptionist.projects.map { |name, api_key| Project.new(name) } end + + def ==(other) + name == other.name + end + + def inspect + "(Project name=#{name})" + end end From 42d223d40270d92db0b383bdc2a978fed2f3bbdf Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:42:47 +0200 Subject: [PATCH 228/293] use helper function; is always type date --- lib/models/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index f88a603..82527fc 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -53,7 +53,7 @@ def to_hash api_key: api_key, version: version, changelog_link: changelog_link, - occurred_at: occurred_at.is_a?(String) ? occurred_at : occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + occurred_at: Helper.es_time(occurred_at) } end end From 9751ba6d357d6a2bee022d836c7b774a3387693c Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 14:47:32 +0200 Subject: [PATCH 229/293] use helper function for dates --- lib/models/deploy.rb | 2 +- lib/models/mailer.rb | 2 +- lib/models/uber_exception.rb | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 82527fc..582be49 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -10,7 +10,7 @@ def initialize(attributes={}) end def self.find_by_project_since(project, date) - find( filters: [ { term: { project_name: project } }, { range: { occurred_at: { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ] ) + find( filters: [ { term: { project_name: project } }, { range: { occurred_at: { gte: Helper.es_time(date) } } } ] ) end def self.find_by_project(project) diff --git a/lib/models/mailer.rb b/lib/models/mailer.rb index b1c9417..6757539 100644 --- a/lib/models/mailer.rb +++ b/lib/models/mailer.rb @@ -8,7 +8,7 @@ def self.deliver_new_exceptions(project, day, to_address, body) To: Exceptionist MIME-Version: 1.0 Content-type: text/html -Subject: [Exceptionist][#{project.name}] Summary for #{day.strftime('%Y-%m-%d')} +Subject: [Exceptionist][#{project.name}] Summary for #{Helper.es_day(day)} #{body} MESSAGE_END diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 8c3fc29..587ed71 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -18,7 +18,7 @@ def self.count_all(project) end def self.count_since(project: '', date: '') - Exceptionist.esclient.count(type: TYPE_EXCEPTIONS, filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { gte: date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + Exceptionist.esclient.count(type: TYPE_EXCEPTIONS, filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { gte: Helper.es_time(date) } } ] ) end def self.get(uber_key) @@ -70,7 +70,7 @@ def self.aggregation_since_last_deploy(project) deploy = Deploy.find_last_deploy(project) raise 'There is no deploy' if deploy.nil? - filters_occur = [{ term: { project_name: project } }, { range: { occurred_at: { gte: deploy.occurred_at.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } } ] + filters_occur = [{ term: { project_name: project } }, { range: { occurred_at: { gte: Helper.es_time(deploy.occurred_at) } } } ] Exceptionist.esclient.search_aggs( filters_occur,'uber_key') end @@ -92,7 +92,7 @@ def self.occurred(occurrence) end - first_timestamp = first_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%L%z") + first_timestamp = Helper.es_time(first_timestamp) hash = occurrence.to_hash hash[:id] = occurrence.id Exceptionist.esclient.update(id: occurrence.uber_key, body: { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', @@ -105,7 +105,7 @@ def self.forget_old_exceptions(project, days=0) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { lte: since_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z") } } ] ) + uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { lte: Helper.es_time(since_date) } } ] ) uber_exceptions.each do |exception| exception.forget! From a4a0a6b2ce01cd3332df9a78401c6687b2940d47 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:04:10 +0200 Subject: [PATCH 230/293] move search deploys into deploy class --- lib/es_client.rb | 22 ++++++---------------- lib/helper.rb | 7 +++++++ lib/models/deploy.rb | 5 ++++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index f236afb..1405fdf 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -5,16 +5,18 @@ class ESClient INDEX = 'exceptionist' TYPE_EXCEPTIONS = 'exceptions' TYPE_OCCURRENCES = 'occurrences' - TYPE_DEPLOYS = 'deploys' def initialize(endpoint) @host, @port = endpoint.split(':') @es = Elasticsearch::Client.new(host: endpoint) end - def search_deploys(filters: {}, sort: {}, from: 0, size: 25) - hash = search(type: TYPE_DEPLOYS, filters: filters, sort: sort, from: from, size: size) - hash.hits.hits.map { |doc| create_deploy(doc) } + def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) + raise ArgumentError, 'from has to be >= 0' if from < 0 + + query = create_search_query(filters, sort, from, size) + response = @es.search(index: INDEX, type: type, body: query) + Hashie::Mash.new(response) end def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) @@ -90,13 +92,6 @@ def refresh end private - def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) - raise ArgumentError, 'from has to be >= 0' if from < 0 - - query = create_search_query(filters, sort, from, size) - response = @es.search(index: INDEX, type: type, body: query) - Hashie::Mash.new(response) - end def create_occurrence(attr) attr = transform(attr) @@ -108,11 +103,6 @@ def create_exception(attr) UberException.new(attr) end - def create_deploy(attr) - attr = transform(attr) - Deploy.new(attr) - end - def transform(attr) attr.merge!(attr['_source']).delete('_source') attr = Helper.symbolize_keys(attr) diff --git a/lib/helper.rb b/lib/helper.rb index 345e286..ab7ed19 100644 --- a/lib/helper.rb +++ b/lib/helper.rb @@ -33,4 +33,11 @@ def self.es_time(date) def self.es_day(date) date.strftime('%Y-%m-%d') end + + def self.transform(attr) + attr.merge!(attr['_source']).delete('_source') + attr = Helper.symbolize_keys(attr) + attr[:id] = attr.delete :_id + attr + end end diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 582be49..7eeb2a8 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -1,6 +1,8 @@ class Deploy attr_accessor :id, :project_name, :api_key, :version, :changelog_link, :occurred_at + TYPE_DEPLOYS = 'deploys' + def initialize(attributes={}) attributes.each do |key, value| instance_variable_set("@#{key}", value) @@ -22,7 +24,8 @@ def self.find_last_deploy(project) end def self.find(filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) - Exceptionist.esclient.search_deploys( filters: filters, sort: sort, from: from, size: size ) + hash = Exceptionist.esclient.search(type: TYPE_DEPLOYS, filters: filters, sort: sort, from: from, size: size) + hash.hits.hits.map { |doc| new(Helper.transform(doc)) } end def self.from_json(json) From e9fa34f395a65eaf951be1df87a04adbe9471727 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:10:10 +0200 Subject: [PATCH 231/293] move search occurrence to occurrence class --- lib/es_client.rb | 12 +----------- lib/models/occurrence.rb | 5 ++++- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 1405fdf..22e36ff 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -11,7 +11,7 @@ def initialize(endpoint) @es = Elasticsearch::Client.new(host: endpoint) end - def search(type: 'occurrences', filters: {}, sort: {}, from: 0, size: 25) + def search(type: '', filters: {}, sort: {}, from: 0, size: 25) raise ArgumentError, 'from has to be >= 0' if from < 0 query = create_search_query(filters, sort, from, size) @@ -24,11 +24,6 @@ def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) hash.hits.hits.map { |doc| create_exception(doc) } end - def search_occurrences(filters: {}, sort: {}, from: 0, size: 25) - hash = search(filters: filters, sort: sort, from: from, size: size) - hash.hits.hits.map { |doc| create_occurrence(doc) } - end - def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } @@ -93,11 +88,6 @@ def refresh private - def create_occurrence(attr) - attr = transform(attr) - Occurrence.new(attr) - end - def create_exception(attr) attr = transform(attr) UberException.new(attr) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 87ae5af..5e1ae7c 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -4,6 +4,7 @@ class Occurrence :parameters, :session, :cgi_data, :environment, :project_name, :occurred_at, :occurred_at_day, :id, :uber_key, :api_key, :sort + TYPE_OCCURRENCES = 'occurrences' def initialize(attributes={}) attributes.each do |key, value| @@ -43,7 +44,9 @@ def self.find_next(uber_key, date) def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } }, from: 0, size: 25) filters = Helper.wrap(filters) filters << { term: { uber_key: uber_key } } unless uber_key.empty? - Exceptionist.esclient.search_occurrences( filters: filters, sort: sort, from: from, size: size ) + + hash = Exceptionist.esclient.search(type: TYPE_OCCURRENCES, filters: filters, sort: sort, from: from, size: size) + hash.hits.hits.map { |doc| new(Helper.transform(doc)) } end def self.count_all_on(project, day) From b30b0134a1e7490a26e3155721761737a39dcfa6 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:18:52 +0200 Subject: [PATCH 232/293] move search exception into exception class --- lib/es_client.rb | 5 ----- lib/models/uber_exception.rb | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 22e36ff..64562e9 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -19,11 +19,6 @@ def search(type: '', filters: {}, sort: {}, from: 0, size: 25) Hashie::Mash.new(response) end - def search_exceptions(filters: {}, sort: {}, from: 0, size: 25) - hash = search(type: TYPE_EXCEPTIONS, filters: filters, sort: sort, from: from, size: size) - hash.hits.hits.map { |doc| create_exception(doc) } - end - def search_aggs(filters, aggs) # size set 0 for Integer.MAX_VALUE query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 587ed71..639b807 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -77,7 +77,9 @@ def self.aggregation_since_last_deploy(project) def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) terms = terms.map { |term| { term: term } unless term.nil? } terms << { term: { closed: false } } - Exceptionist.esclient.search_exceptions(filters: terms.push(*filters), sort: sort, from: from, size: size) + + hash = Exceptionist.esclient.search(type: TYPE_EXCEPTIONS, filters: terms.push(*filters), sort: sort, from: from, size: size) + hash.hits.hits.map { |doc| new(Helper.transform(doc)) } end def self.occurred(occurrence) @@ -105,7 +107,7 @@ def self.forget_old_exceptions(project, days=0) since_date = Time.now - (86400 * days) deleted = 0 - uber_exceptions = Exceptionist.esclient.search_exceptions( filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { lte: Helper.es_time(since_date) } } ] ) + uber_exceptions = find( filters: [ { term: { project_name: project } }, range: { 'last_occurrence.occurred_at' => { lte: Helper.es_time(since_date) } } ] ) uber_exceptions.each do |exception| exception.forget! From ef1b97ac8f7060c5f1062e6f2339ef87b9b0b2e3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:25:37 +0200 Subject: [PATCH 233/293] make search_aggs more generic --- lib/es_client.rb | 4 ++-- lib/models/occurrence.rb | 4 ++++ lib/models/uber_exception.rb | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 64562e9..98e3b49 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -19,10 +19,10 @@ def search(type: '', filters: {}, sort: {}, from: 0, size: 25) Hashie::Mash.new(response) end - def search_aggs(filters, aggs) + def search_aggs(type: '', filters: [], aggs: '') # size set 0 for Integer.MAX_VALUE query = { query: wrap_filters(filters), aggs: { exceptions: { terms: { field: aggs, size: 0 } } } } - response = @es.search(index: INDEX, type: TYPE_OCCURRENCES, body: query, search_type: 'count') + response = @es.search(index: INDEX, type: type, body: query, search_type: 'count') hash = Hashie::Mash.new(response) hash.aggregations.exceptions.buckets end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index 5e1ae7c..f024c13 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -63,6 +63,10 @@ def self.count(project: '', filters: {}) Exceptionist.esclient.count( filters: filters ) end + def self.search_aggs(filters: [], aggs: '') + Exceptionist.esclient.search_aggs(type: TYPE_OCCURRENCES, filters: filters, aggs: aggs) + end + # # accessors # diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 639b807..e1916ab 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -71,7 +71,7 @@ def self.aggregation_since_last_deploy(project) raise 'There is no deploy' if deploy.nil? filters_occur = [{ term: { project_name: project } }, { range: { occurred_at: { gte: Helper.es_time(deploy.occurred_at) } } } ] - Exceptionist.esclient.search_aggs( filters_occur,'uber_key') + Occurrence.search_aggs(filters: filters_occur, aggs: 'uber_key') end def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) From 8621c284ca8199aed189fcb59540ca73c2697f17 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:35:35 +0200 Subject: [PATCH 234/293] dont use default values for function call --- lib/es_client.rb | 14 ++++++-------- lib/models/occurrence.rb | 2 +- lib/models/uber_exception.rb | 10 +++++----- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 98e3b49..3be29b3 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -3,8 +3,6 @@ class ESClient attr_accessor :es, :host, :port INDEX = 'exceptionist' - TYPE_EXCEPTIONS = 'exceptions' - TYPE_OCCURRENCES = 'occurrences' def initialize(endpoint) @host, @port = endpoint.split(':') @@ -33,22 +31,22 @@ def search_ids(type: TYPE_EXCEPTIONS, ids: []) hash.hits.hits.map { |doc| create_exception(doc) } end - def mget(ids: []) - response = @es.mget( index: INDEX, type: TYPE_EXCEPTIONS, body: { ids: ids } ) + def mget(type: '', ids: []) + response = @es.mget( index: INDEX, type: type, body: { ids: ids } ) hash = Hashie::Mash.new(response) hash.docs.map { |doc| create_exception(doc) } end - def index(type: TYPE_OCCURRENCES, body: {}) + def index(type: '', body: {}) response = @es.index(index: INDEX, type: type, body: body) Hashie::Mash.new(response) end - def update(type: TYPE_EXCEPTIONS, id: -1, body: {}) + def update(type: '', id: -1, body: {}) @es.update(index: INDEX, type: type, id: id, body: body) end - def count(type: TYPE_OCCURRENCES, filters: []) + def count(type: '', filters: []) @es.count(index: INDEX, type: type, body: { query: wrap_filters(filters) } )['count'] end @@ -56,7 +54,7 @@ def delete_by_query(query: { match_all: {} }) @es.delete_by_query( index: INDEX, body: { query: query } ) end - def delete(type: TYPE_EXCEPTIONS, id: -1) + def delete(type: '', id: -1) @es.delete(index: INDEX, type: type, id: id) end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index f024c13..b5e357a 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -60,7 +60,7 @@ def self.count_since(uber_key, date) def self.count(project: '', filters: {}) filters = Helper.wrap(filters) filters << { term: { project_name: project } } unless project.empty? - Exceptionist.esclient.count( filters: filters ) + Exceptionist.esclient.count( type: TYPE_OCCURRENCES, filters: filters ) end def self.search_aggs(filters: [], aggs: '') diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index e1916ab..70b352b 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -52,7 +52,7 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', catego ids = [] agg_exces.each { |occurr| ids << occurr['key'] } - exces = Exceptionist.esclient.mget(ids: ids) + exces = Exceptionist.esclient.mget(type: TYPE_EXCEPTIONS, ids: ids) exces.select!{ |exce| exce.category == category && !exce.closed } unless category.nil? exces.slice(from, size).each do |exce| agg_exces.each do |occurr| @@ -97,7 +97,7 @@ def self.occurred(occurrence) first_timestamp = Helper.es_time(first_timestamp) hash = occurrence.to_hash hash[:id] = occurrence.id - Exceptionist.esclient.update(id: occurrence.uber_key, body: { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', + Exceptionist.esclient.update(type: TYPE_EXCEPTIONS, id: occurrence.uber_key, body: { script: 'ctx._source.occurrences_count += 1; ctx._source.closed=false; ctx._source.last_occurrence=occurrence; ctx._source.first_occurred_at=timestamp', upsert: { project_name: occurrence.project_name, last_occurrence: hash, first_occurred_at: first_timestamp, closed: false, occurrences_count: 1, category: 'no-category'}, params: { occurrence: hash, timestamp: first_timestamp} }) UberException.get(occurrence.uber_key) @@ -120,15 +120,15 @@ def self.forget_old_exceptions(project, days=0) def forget! Occurrence.delete_all_for(id) - Exceptionist.esclient.delete(id: id) + Exceptionist.esclient.delete(type: TYPE_EXCEPTIONS, id: id) end def close! - Exceptionist.esclient.update(id: @id, body: { doc: { closed: true } }) + Exceptionist.esclient.update(type: TYPE_EXCEPTIONS, id: @id, body: { doc: { closed: true } }) end def update(doc) - Exceptionist.esclient.update(id: @id, body: { doc: doc }) + Exceptionist.esclient.update(type: TYPE_EXCEPTIONS, id: @id, body: { doc: doc }) end def current_occurrence(position) From 56ed48d32af447f4ea44368b7755e54572f184ee Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:36:06 +0200 Subject: [PATCH 235/293] remove unused function --- lib/es_client.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 3be29b3..4b78721 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -25,12 +25,6 @@ def search_aggs(type: '', filters: [], aggs: '') hash.aggregations.exceptions.buckets end - def search_ids(type: TYPE_EXCEPTIONS, ids: []) - response = @es.search(index: INDEX, type: type, body: { query: { ids: { values: ids } } } ) - hash = Hashie::Mash.new(response) - hash.hits.hits.map { |doc| create_exception(doc) } - end - def mget(type: '', ids: []) response = @es.mget( index: INDEX, type: type, body: { ids: ids } ) hash = Hashie::Mash.new(response) From 215cb65190058461e1273a12e1f121faa409081f Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:40:08 +0200 Subject: [PATCH 236/293] remove model related stuff from esclient --- lib/es_client.rb | 11 ++--------- lib/models/uber_exception.rb | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index 4b78721..e97ec42 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -28,7 +28,7 @@ def search_aggs(type: '', filters: [], aggs: '') def mget(type: '', ids: []) response = @es.mget( index: INDEX, type: type, body: { ids: ids } ) hash = Hashie::Mash.new(response) - hash.docs.map { |doc| create_exception(doc) } + hash.docs end def index(type: '', body: {}) @@ -65,8 +65,7 @@ def get_mapping(type) end def get(type: '', id: id) - response = @es.get(index: INDEX, type: type, id: id) - create_exception(response) + @es.get(index: INDEX, type: type, id: id) end def refresh @@ -74,12 +73,6 @@ def refresh end private - - def create_exception(attr) - attr = transform(attr) - UberException.new(attr) - end - def transform(attr) attr.merge!(attr['_source']).delete('_source') attr = Helper.symbolize_keys(attr) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 70b352b..77ee2a1 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -22,7 +22,7 @@ def self.count_since(project: '', date: '') end def self.get(uber_key) - Exceptionist.esclient.get(type: TYPE_EXCEPTIONS, id: uber_key) + new(Helper.transform(Exceptionist.esclient.get(type: TYPE_EXCEPTIONS, id: uber_key))) end def self.find_sorted_by_occurrences_count(terms: [], from: 0, size: 25) @@ -52,7 +52,7 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', catego ids = [] agg_exces.each { |occurr| ids << occurr['key'] } - exces = Exceptionist.esclient.mget(type: TYPE_EXCEPTIONS, ids: ids) + exces = Exceptionist.esclient.mget(type: TYPE_EXCEPTIONS, ids: ids).map { |doc| new(Helper.transform(doc)) } exces.select!{ |exce| exce.category == category && !exce.closed } unless category.nil? exces.slice(from, size).each do |exce| agg_exces.each do |occurr| From e03dc1c73008fed7ef2329bc42991393c476788a Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:40:46 +0200 Subject: [PATCH 237/293] remove anused function --- lib/es_client.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index e97ec42..bfacfb2 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -73,13 +73,6 @@ def refresh end private - def transform(attr) - attr.merge!(attr['_source']).delete('_source') - attr = Helper.symbolize_keys(attr) - attr[:id] = attr.delete :_id - attr - end - def create_search_query(filters, sort, from, size) { query: wrap_filters(filters), sort: sort, from: from, size: size } end From 78de4ec143f277ff3caded1a31caa468fb56d264 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 15:43:37 +0200 Subject: [PATCH 238/293] use constants instead of strings --- lib/models/deploy.rb | 2 +- lib/models/occurrence.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/deploy.rb b/lib/models/deploy.rb index 7eeb2a8..36d8ca6 100644 --- a/lib/models/deploy.rb +++ b/lib/models/deploy.rb @@ -36,7 +36,7 @@ def self.from_json(json) end def save - deploy = Exceptionist.esclient.index(type: 'deploys', body: to_hash) + deploy = Exceptionist.esclient.index(type: TYPE_DEPLOYS, body: to_hash) @id = deploy._id self end diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index b5e357a..fcf390a 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -101,7 +101,7 @@ def project # def save - occurrence = Exceptionist.esclient.index(type: 'occurrences', body: to_hash) + occurrence = Exceptionist.esclient.index(type: TYPE_OCCURRENCES, body: to_hash) @id = occurrence._id self end From 41d34387a883d390f1ca494a0e83330170036b82 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 16:08:11 +0200 Subject: [PATCH 239/293] refactor since last deploy functions --- lib/models/uber_exception.rb | 46 ++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 77ee2a1..7304125 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -30,31 +30,35 @@ def self.find_sorted_by_occurrences_count(terms: [], from: 0, size: 25) end def self.find_since_last_deploy(project: '', terms: [], from: 0, size: 25) - agg_exces = aggregation_since_last_deploy(project) + agg_exces, ids = aggregation_since_last_deploy(project) - ids = [] - agg_exces.each { |occurr| ids << occurr['key'] } exces = find( terms: terms.compact << { closed: false }, filters: [ { ids: { type: TYPE_EXCEPTIONS, values: ids } } ], from: from, size: size ) - exces.each do |exce| - agg_exces.each do |occurr| - if occurr['key'] == exce.id - exce.occurrences_count = occurr['doc_count'] - agg_exces.delete(occurr) - break - end - end - end - exces + merge(exces, agg_exces) end def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', category: nil, from: 0, size: 25) - agg_exces = aggregation_since_last_deploy(project) + agg_exces, ids = aggregation_since_last_deploy(project) - ids = [] - agg_exces.each { |occurr| ids << occurr['key'] } + # to preserve ordering and filtering category at the same time, filtering has to be done in ruby, not on db-level exces = Exceptionist.esclient.mget(type: TYPE_EXCEPTIONS, ids: ids).map { |doc| new(Helper.transform(doc)) } exces.select!{ |exce| exce.category == category && !exce.closed } unless category.nil? - exces.slice(from, size).each do |exce| + merge(exces.slice(from, size), agg_exces) + end + + def self.aggregation_since_last_deploy(project) + deploy = Deploy.find_last_deploy(project) + raise 'There is no deploy' if deploy.nil? + + filters_occur = [{ term: { project_name: project } }, { range: { occurred_at: { gte: Helper.es_time(deploy.occurred_at) } } } ] + agg_exces = Occurrence.search_aggs(filters: filters_occur, aggs: 'uber_key') + ids = [] + agg_exces.each { |occurr| ids << occurr['key'] } + + return agg_exces, ids + end + + def self.merge(exces, agg_exces) + exces.each do |exce| agg_exces.each do |occurr| if occurr['key'] == exce.id exce.occurrences_count = occurr['doc_count'] @@ -66,14 +70,6 @@ def self.find_since_last_deploy_ordered_by_occurrences_count(project: '', catego exces end - def self.aggregation_since_last_deploy(project) - deploy = Deploy.find_last_deploy(project) - raise 'There is no deploy' if deploy.nil? - - filters_occur = [{ term: { project_name: project } }, { range: { occurred_at: { gte: Helper.es_time(deploy.occurred_at) } } } ] - Occurrence.search_aggs(filters: filters_occur, aggs: 'uber_key') - end - def self.find(terms: [], filters: [], sort: { 'last_occurrence.occurred_at' => { order: 'desc'} }, from: 0, size: 25) terms = terms.map { |term| { term: term } unless term.nil? } terms << { term: { closed: false } } From 20d8dee938b4864e0aca2aa62915a083f855545e Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 16:12:22 +0200 Subject: [PATCH 240/293] remove unused function --- lib/models/project.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/models/project.rb b/lib/models/project.rb index 3994532..43e5727 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -22,10 +22,6 @@ def deploys_last_thirty_days Deploy.find_by_project_since(@name, since) end - def total_count_on(day) - Occurrence.count_all_on(name, day) - end - def self.find_by_key(api_key) project = Exceptionist.projects.find { |name, project_key| project_key == api_key } project ? Project.new(project.first) : nil From 9b3210d9772c9b4e45ef9b6e661df5f839a7d756 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 16:12:45 +0200 Subject: [PATCH 241/293] add test for deploy page --- test/integration_test.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration_test.rb b/test/integration_test.rb index fd3a2a1..27724d7 100644 --- a/test/integration_test.rb +++ b/test/integration_test.rb @@ -275,4 +275,8 @@ def test_projects_be_able_to_close_an_exception assert_not_contain 'NameError in users#show' assert_contain 'NameError in users#delete' end + + def test_deploys + visit '/deploys/ExampleProject' + end end From 4e1495c7b75029e18973adde6646673b20014fc3 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Tue, 23 Sep 2014 16:15:38 +0200 Subject: [PATCH 242/293] reenable wrap sort --- lib/es_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/es_client.rb b/lib/es_client.rb index bfacfb2..b0ae40f 100644 --- a/lib/es_client.rb +++ b/lib/es_client.rb @@ -74,7 +74,7 @@ def refresh private def create_search_query(filters, sort, from, size) - { query: wrap_filters(filters), sort: sort, from: from, size: size } + { query: wrap_filters(filters), sort: wrap_sort(sort), from: from, size: size } end def wrap_sort(sort) From 22b5202433d97c7810924a940d536944f71be2c5 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 24 Sep 2014 10:04:26 +0200 Subject: [PATCH 243/293] change color of low category --- public/stylesheets/exceptionist.css | 16 ++++++++-------- views/_occurrence.erb | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/public/stylesheets/exceptionist.css b/public/stylesheets/exceptionist.css index 8f82f12..f496767 100644 --- a/public/stylesheets/exceptionist.css +++ b/public/stylesheets/exceptionist.css @@ -111,7 +111,7 @@ span.new { } .low { - background-color: #F5F8E5; + background-color: #EFF5F9; } .high { @@ -180,7 +180,7 @@ button img, a.button img{ /* Standard */ button:hover, a.button:hover{ - background-color:#dff4ff; + background-color:#C9DCE6; border:1px solid #c2e1ef; color:#336699; } @@ -190,6 +190,12 @@ a.button:active{ color:#fff; } +.active { + background-color:#dff4ff; + border:1px solid #c2e1ef; + color:#336699; +} + /* Positive */ body .positive { color:#529214; @@ -201,12 +207,6 @@ a.positive:hover, button.positive:hover{ color:#529214; } -.positive.active { - background-color:#E6EFC2; - border:1px solid #C6D880; - color:#529214; -} - /* Negative */ body .negative { color:#d12f19; diff --git a/views/_occurrence.erb b/views/_occurrence.erb index 98b7f6e..1501fb7 100644 --- a/views/_occurrence.erb +++ b/views/_occurrence.erb @@ -39,7 +39,7 @@ - +
From ee9d162d319751b7e4b7cf33f627dfa68647f8c7 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 24 Sep 2014 13:15:58 +0200 Subject: [PATCH 244/293] make user changes happen immediately --- lib/app.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/app.rb b/lib/app.rb index 57af37d..8677110 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -110,6 +110,7 @@ class ExceptionistApp < Sinatra::Base post '/exceptions/:id/close' do @uber_exceptions = UberException.get(params[:id]) @uber_exceptions.close! + Exceptionist.esclient.refresh redirect to("/projects/#{@uber_exceptions.project_name}?#{Rack::Utils.unescape(params[:backparams])}") end @@ -117,6 +118,7 @@ class ExceptionistApp < Sinatra::Base post '/exceptions/category' do @uber_exceptions = UberException.get(params[:id]) @uber_exceptions.update( category: params[:category] ) + Exceptionist.esclient.refresh halt 200, 'category changed' end From 2067cb9b017f4e96680455a955531dc43acd6817 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 24 Sep 2014 14:08:24 +0200 Subject: [PATCH 245/293] fix wrong occurence count in exception graph --- lib/models/occurrence.rb | 4 ---- lib/models/project.rb | 2 +- lib/models/uber_exception.rb | 2 +- test/occurrence_test.rb | 5 ----- test/uber_exception_test.rb | 2 +- 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/models/occurrence.rb b/lib/models/occurrence.rb index fcf390a..2760d52 100644 --- a/lib/models/occurrence.rb +++ b/lib/models/occurrence.rb @@ -49,10 +49,6 @@ def self.find(uber_key: '', filters: {}, sort: { occurred_at: { order: 'desc' } hash.hits.hits.map { |doc| new(Helper.transform(doc)) } end - def self.count_all_on(project, day) - Occurrence.count( project: project, filters: { term: { occurred_at_day: Helper.es_day(day) } }) - end - def self.count_since(uber_key, date) Occurrence.count(filters: [{ range: { occurred_at: { gte: Helper.es_time(date) } } }, { term: { uber_key: uber_key } }] ) end diff --git a/lib/models/project.rb b/lib/models/project.rb index 43e5727..735ea5a 100644 --- a/lib/models/project.rb +++ b/lib/models/project.rb @@ -10,7 +10,7 @@ def exceptions_count end def last_thirty_days - Helper.last_n_days(30).map { |day| [day, Occurrence.count_all_on(name,day)] } + Helper.last_n_days(30).map { |day| [day, Occurrence.count(filters: [ { term: { project_name: name } }, { term: { occurred_at_day: Helper.es_day(day) } } ] )] } end def last_deploy diff --git a/lib/models/uber_exception.rb b/lib/models/uber_exception.rb index 7304125..4b83f3a 100644 --- a/lib/models/uber_exception.rb +++ b/lib/models/uber_exception.rb @@ -150,7 +150,7 @@ def url end def occurrences_count_on(date) - Occurrence.count_all_on(project_name, date) + Occurrence.count( filters: [ { term: { uber_key: @id } }, { term: { occurred_at_day: Helper.es_day(date) } } ] ) end def last_thirty_days diff --git a/test/occurrence_test.rb b/test/occurrence_test.rb index beec499..9f25870 100644 --- a/test/occurrence_test.rb +++ b/test/occurrence_test.rb @@ -63,11 +63,6 @@ def test_find_next assert_equal @occur12, Occurrence.find_next(@occur11.uber_key, Time.local(2011, 1, 1, 12, 0)) end - def test_count_all_on - assert_equal 0, Occurrence.count_all_on('ExampleProject', Time.local(2011, 1, 20)) - assert_equal 4, Occurrence.count_all_on('ExampleProject', Time.local(2011, 1, 6)) - end - def test_count_since assert_equal 2, Occurrence.count_since(@occur11.uber_key, Time.local(2011, 1, 7, 12, 0)) assert_equal 0, Occurrence.count_since(@occur21.uber_key, Time.local(2011, 1, 8)) diff --git a/test/uber_exception_test.rb b/test/uber_exception_test.rb index e53ede9..27baf57 100644 --- a/test/uber_exception_test.rb +++ b/test/uber_exception_test.rb @@ -162,7 +162,7 @@ def test_current_occurrence def test_occurrences_count_on assert_equal 1, @exce1.occurrences_count_on(Time.local(2011, 1, 9)) - assert_equal 3, @exce2.occurrences_count_on(Time.local(2011, 1, 4)) + assert_equal 4, @exce2.occurrences_count_on(Time.local(2011, 1, 6)) end def test_new_since From fe38c277380a7e9dfdb316168732a4d380bdf241 Mon Sep 17 00:00:00 2001 From: Simon Mulser Date: Wed, 24 Sep 2014 14:12:51 +0200 Subject: [PATCH 246/293] rename variable --- views/_thirty_day_graph.erb | 4 ++-- views/dashboard.erb | 2 +- views/show.erb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/views/_thirty_day_graph.erb b/views/_thirty_day_graph.erb index 643e57c..d380e7c 100644 --- a/views/_thirty_day_graph.erb +++ b/views/_thirty_day_graph.erb @@ -1,11 +1,11 @@