better handling for application statistics

.gitignore

@@ -6,6 +6,7 @@
 .bundle
 .config
 .yardoc
+.rvmrc
 Gemfile.lock
 InstalledFiles
 _yardoc

ChangeLog

@@ -1,3 +1,10 @@
+2013/12/30 : Sho Sawada
+
+	* ver x.x.x
+	* enhanced Application Statistics report API
+	* you can describe configuration in any context (like aws/aws-sdk-ruby)
+	* use WebMock in spec test
+
 2013/8/27 : Takuya Murakami
 
 	* ver 1.0.3

README.ja.md

@@ -47,7 +47,7 @@ Google Play メールアドレスとパスワード、デベロッパIDを設定
 デベロッパID は、developer console にログインした後の URL 末尾の
 dev_acc=... の数字です。
 
-```
+```yaml
 # GooglePlay dev scraper config file sample (YAML format)
 #
 # Place this content to your ~/.googleplay_dev_scraper or
@@ -120,22 +120,75 @@ API の利用
 
 例:
 
-```
+```ruby
 require 'googleplay_dev_scraper'
 
 scraper = GooglePlayDevScraper::Scraper.new
 
-# set config (Note: config file is not read via API access)
-scraper.config.email = "foo@example.com"
-scraper.config.password = "YOUR_PASSWORD"
-scraper.config.dev_acc = "1234567890"
+# 設定 (config ファイルはモジュールを用いた場合には読み込まれません)
+GooglePlayDevScraper.config(
+  email: "foo@example.com"
+  password: "YOUR_PASSWORD"
+  dev_acc: "1234567890"
+)
 
-# get sales report / estimated sales report
+# 売上レポート取得
 puts scraper.get_sales_report(2012, 11)
 puts scraper.get_estimated_sales_report(2012, 12)
 
-# get orders
-puts scraper.get_order_list(DateTime.parse("2012-11-01"), DateTime.parse("2012-11-30"))
+# オーダー一覧取得
+puts scraper.get_order_list(
+  DateTime.parse("2012-11-01 00:00:00"), DateTime.parse("2012-11-30 23:59:59")
+)
+
+# アプリケーション統計情報取得
+#   dimensions オプションに利用可能なパラメータ:
+#     overall os_version device country language app_version carrier
+#     gcm_message_status gcm_response_code crash_details anr_details
+#
+#    ('device' dimension をセットすると、返却されるデータがとても大きくなります!!) 
+#
+#   metrics オプションに利用可能なパラメータ:
+#     current_device_installs daily_device_installs daily_device_uninstalls
+#     daily_device_upgrades current_user_installs total_user_installs
+#     daily_user_installs daily_user_uninstalls daily_avg_rating total_avg_rating
+#     gcm_messages gcm_registrations daily_crashes daily_anrs
+
+stats = GooglePlayDevScraper::ApplicationStatistics.fetch(
+  'com.example.helloworld',
+  dimensions: %w(os_version country),
+  metrics: %w(total_user_installs current_device_installs)
+  start_date: Date.new(2013, 12, 5),
+  end_date: Date.new(2013, 12, 6)
+)
+
+# stats は "GooglePlayDevScraper::ApplicationStatistics" クラスの配列です 
+stats[0].date
+# => 2013-12-05
+
+stats[0].dimension
+# => :os_version
+
+stats[0].field_name
+# => current_device_installs
+
+stats[0].entries
+# => {"Android 2.3" => 1234, "Android 2.2" => 321 ....}
+
+# 'select_by' と 'find_by' メソッドを用意しました。条件を指定して探せます。
+
+stats.select_by(date: Date.new(2013, 12, 6), dimension: :os_version)
+# =>
+# [
+#   #<GooglePlayDevScraper::ApplicationStatistics, @date=#<Date: 2013-12-06> .. >,
+#   #<GooglePlayDevScraper::ApplicationStatistics, @date=#<Date: 2013-12-06> .. >,
+#   .. 
+# ]
+
+stats.find_by(date: Date.new(2013, 12, 6), dimension: :os_version)
+# => #<GooglePlayDevScraper::ApplicationStatistics .. >
+# ( 'find_by' method always returns only one object which matches first )
+
 ```
 
 内部動作とか

README.md

@@ -37,7 +37,7 @@ Configuration
 Create configuration file at ~/.googleplay_dev_scraper,
 or ./.googleplay_dev_scraper in YAML format.
 
-```
+```yaml
 # GooglePlay scraper config file sample (YAML format)
 #
 # Place this content to your ~/.googleplay_dev_scraper or
@@ -103,22 +103,76 @@ API usage
 
 Example:
 
-```
+```ruby
 require 'googleplay_dev_scraper'
 
-scraper = GooglePlayDevScraper::Scraper.new
-
 # set config (Note: config file is not read via API access)
-scraper.config.email = "foo@example.com"
-scraper.config.password = "YOUR_PASSWORD"
-scraper.config.dev_acc = "1234567890"
+GooglePlayDevScraper.config(
+  email: "foo@example.com"
+  password: "YOUR_PASSWORD"
+  dev_acc: "1234567890"
+)
+
+scraper = GooglePlayDevScraper::Scraper.new
 
 # get sales report / estimated sales report
 puts scraper.get_sales_report(2012, 11)
 puts scraper.get_estimated_sales_report(2012, 12)
 
 # get orders
-puts scraper.get_order_list(DateTime.parse("2012-11-01 00:00:00", DateTime.parse("2012-11-30T23:59:59"))
+puts scraper.get_order_list(
+  DateTime.parse("2012-11-01 00:00:00"), DateTime.parse("2012-11-30 23:59:59")
+)
+
+# get application statistics
+#   available dimensions :
+#     overall os_version device country language app_version carrier
+#     gcm_message_status gcm_response_code crash_details anr_details
+#
+#    (use 'device' dimension may returns huge data set. be careful!) 
+#
+#   available metrics :
+#     current_device_installs daily_device_installs daily_device_uninstalls
+#     daily_device_upgrades current_user_installs total_user_installs
+#     daily_user_installs daily_user_uninstalls daily_avg_rating total_avg_rating
+#     gcm_messages gcm_registrations daily_crashes daily_anrs
+
+stats = GooglePlayDevScraper::ApplicationStatistics.fetch(
+  'com.example.helloworld',
+  dimensions: %w(os_version country),
+  metrics: %w(total_user_installs current_device_installs)
+  start_date: Date.new(2013, 12, 5),
+  end_date: Date.new(2013, 12, 6)
+)
+
+# stats is array of "GooglePlayDevScraper::ApplicationStatistics" class
+stats[0].date
+# => 2013-12-05
+
+stats[0].dimension
+# => :os_version
+
+stats[0].field_name
+# => current_device_installs
+
+stats[0].entries
+# => {"Android 2.3" => 1234, "Android 2.2" => 321 ....}
+
+# to find object, 'select_by' and 'find_by' may help you.
+
+stats.select_by(date: Date.new(2013, 12, 6), dimension: :os_version)
+# =>
+# [
+#   #<GooglePlayDevScraper::ApplicationStatistics, @date=#<Date: 2013-12-06> .. >,
+#   #<GooglePlayDevScraper::ApplicationStatistics, @date=#<Date: 2013-12-06> .. >,
+#   .. 
+# ]
+
+stats.find_by(date: Date.new(2013, 12, 6), dimension: :os_version)
+# => #<GooglePlayDevScraper::ApplicationStatistics .. >
+# ( 'find_by' method always returns only one object which matches first )
+
+
 ```
 
 License

googleplay_dev_scraper.gemspec

@@ -19,8 +19,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency('mechanize', '>= 2.5.0')
+  gem.add_dependency('rubyzip', '>= 1.0.0')
 
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rdoc'
+  gem.add_development_dependency 'webmock'
 end

lib/googleplay_dev_scraper.rb

@@ -1,4 +1,7 @@
 require "googleplay_dev_scraper/version"
+require "googleplay_dev_scraper/core"
 require "googleplay_dev_scraper/scraper_base"
 require "googleplay_dev_scraper/scraper_config"
 require "googleplay_dev_scraper/scraper"
+require "googleplay_dev_scraper/application_statistics"
+require "googleplay_dev_scraper/application_statistics_collection"

lib/googleplay_dev_scraper/application_statistics.rb

@@ -0,0 +1,106 @@
+class GooglePlayDevScraper::ApplicationStatistics
+  DIMENSIONS = %w(
+    overall os_version device country language app_version carrier
+    gcm_message_status gcm_response_code crash_details anr_details
+  )
+  METRICS = %w(
+    current_device_installs daily_device_installs daily_device_uninstalls
+    daily_device_upgrades current_user_installs total_user_installs
+    daily_user_installs daily_user_uninstalls daily_avg_rating total_avg_rating
+    gcm_messages gcm_registrations daily_crashes daily_anrs
+  )
+  STATISTICS_DOWNLOAD_HOST = 'play.google.com'
+  STATISTICS_DOWNLOAD_PATH = '/apps/publish/statistics/download'
+
+  attr_reader :date, :dimension, :field_name, :entries
+
+  def initialize(options = {})
+    options.each do |attr, value|
+      instance_variable_set("@#{attr}", value)
+    end
+  end
+
+  class << self
+    def fetch(package, options = {})
+      dimensions = options[:dimensions] || DIMENSIONS
+      metrics = options[:metrics] || METRICS
+      uri = URI::HTTPS.build(
+        host: STATISTICS_DOWNLOAD_HOST,
+        path: STATISTICS_DOWNLOAD_PATH,
+        query: URI.encode_www_form(
+          dim: dimensions.join(','),
+          met: metrics.join(','),
+          package: package,
+          sd: (options[:start_date] || Date.today - 7).strftime('%Y%m%d'),
+          ed: (options[:end_date] || Date.today).strftime('%Y%m%d')
+        )
+      )
+      scraper = GooglePlayDevScraper::ScraperBase.new
+      scraper.try_get(uri.to_s)
+
+      case scraper.last_response.content_type
+      when 'text/csv'
+        entries = parse_csv(dimensions.first, scraper.last_response_body)
+      when 'application/zip'
+        tempfile = Tempfile.new('csv.zip')
+        tempfile.binmode
+        tempfile.write(scraper.last_response_body)
+        tempfile.rewind
+        zipfile = Zip::File.new(tempfile.path)
+        entries = zipfile.map {|entry|
+          matches = %r{\A#{Regexp.escape(package)}_(.*)_((installs)|(ratings)|(gcm)|(crashes)).csv\Z}.match(entry.name)
+          if matches && matches[1] && matches[2]
+            entry_stream = zipfile.get_input_stream(entry)
+            parse_csv(matches[1], entry_stream.read)
+          else
+            raise StandardError.new 'unexpected file name'
+          end
+        }.flatten
+      else
+        raise StandardError.new 'unexpected response content type'
+      end
+      GooglePlayDevScraper::ApplicationStatisticsCollection.new(entries)
+    end
+
+    def parse_csv(dimension, csv_data)
+      lines = csv_data.split("\n")
+      4.times{ lines.shift }
+      entries_per_date = {}
+      CSV.parse(lines.join("\n"), headers: true).each do |line|
+        date = line.delete('date').last
+        if entries_per_date.has_key?(date)
+          entries_per_date[date] << line
+        else
+          entries_per_date[date] = [ line ]
+        end
+      end
+      statistics = entries_per_date.map do |date, lines|
+        headers = lines.first.headers.clone
+        primary_key_name = 'overall'
+        if headers.first == dimension
+          primary_key_name = headers.shift
+        end
+        values_per_params = {}
+        headers.each { |field_name| values_per_params[field_name] = Hash.new }
+
+        lines.each do |line|
+          column_name = line[primary_key_name] || primary_key_name
+          headers.each do |field_name|
+            values_per_params[field_name][column_name] = line[field_name].to_i
+          end
+        end
+
+        values_per_params.map do |header, entries|
+          self.new(
+            date: Date.parse(date),
+            dimension: dimension.to_sym,
+            field_name: header.to_sym,
+            entries: entries
+          )
+        end
+      end
+
+      statistics.flatten
+    end
+  end
+end

lib/googleplay_dev_scraper/application_statistics_collection.rb

@@ -0,0 +1,27 @@
+class GooglePlayDevScraper::ApplicationStatisticsCollection < Array
+  def select_by(selector)
+    self.select do |element|
+      ret = true
+      selector.each do |attr, value|
+        if element.instance_variable_get("@#{attr}") != value
+          ret = false
+          break
+        end
+      end
+      ret
+    end
+  end
+
+  def find_by(selector)
+    self.find do |element|
+      ret = true
+      selector.each do |attr, value|
+        if element.instance_variable_get("@#{attr}") != value
+          ret = false
+          break
+        end
+      end
+      ret
+    end
+  end
+end

lib/googleplay_dev_scraper/core.rb

@@ -0,0 +1,25 @@
+module GooglePlayDevScraper
+  class << self
+    @@config = nil
+    @@agent = nil
+
+    def config(options = {})
+      @@config ||= ScraperConfig.new
+      @@config.override_with(options) unless options.empty?
+      @@config
+    end
+
+    def agent
+      @@agent ||= Mechanize.new
+      if config.proxy_host && !config.proxy_host.empty?
+        @@agent.set_proxy(config.proxy_host, config.proxy_port)
+      end
+      @@agent
+    end
+
+    def reset!
+      @@config = nil
+      @@agent = nil
+    end
+  end
+end