Merge pull request #106 from embulk/remove-private_key

Remove deprecated p12 key support

Author: sonots

README.md

@@ -30,9 +30,7 @@ OAuth flow for installed applications.
 | name                                 | type        | required?  | default                  | description            |
 |:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
 |  mode                                | string      | optional   | "append"                 | See [Mode](#mode)     |
-|  auth_method                         | string      | optional   | "private_key"            | `private_key` , `json_key` or `compute_engine`
-|  service_account_email               | string      | required when auth_method is private_key  |   | Your Google service account email
-|  p12_keyfile                         | string      | required when auth_method is private_key  |   | Fullpath of private key in P12(PKCS12) format |
+|  auth_method                         | string      | optional   | "json\_key"              | `json_key`, `compute_engine`, or `application_default`
 |  json_keyfile                        | string      | required when auth_method is json_key     |   | Fullpath of json key |
 |  project                             | string      | required if json_keyfile is not given     |   | project_id |
 |  dataset                             | string      | required   |                          | dataset |
@@ -108,9 +106,8 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
 out:
   type: bigquery
   mode: append
-  auth_method: private_key   # default
-  service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
-  p12_keyfile: /path/to/p12_keyfile.p12
+  auth_method: json_key   # default
+  json_keyfile: /path/to/json_keyfile.json
   project: your-project-000
   dataset: your_dataset_name
   table: your_table_name
@@ -167,22 +164,9 @@ NOTE: BigQuery does not support replacing (actually, copying into) a non-partiti
 
 There are three methods supported to fetch access token for the service account.
 
-1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
-2. JSON key of GCP(Google Cloud Platform)'s service account
-3. Pre-defined access token (Google Compute Engine only)
-
-#### Public-Private key pair of GCP's service account
-
-You first need to create a service account (client ID),
-download its private key and deploy the key with embulk.
-
-```yaml
-out:
-  type: bigquery
-  auth_method: private_key   # default
-  service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
-  p12_keyfile: /path/to/p12_keyfile.p12
-```
+1. JSON key of GCP(Google Cloud Platform)'s service account
+1. Pre-defined access token (Google Compute Engine only)
+1. [Application Default](https://cloud.google.com/docs/authentication/production)
 
 #### JSON key of GCP's service account
 
@@ -211,7 +195,7 @@ out:
        }
 ```
 
-#### Pre-defined access token(GCE only)
+#### Pre-defined access token (GCE only)
 
 On the other hand, you don't need to explicitly create a service account for embulk when you
 run embulk in Google Compute Engine. In this third authentication method, you need to
@@ -224,6 +208,16 @@ out:
   auth_method: compute_engine
 ```
 
+#### Application Default
+
+See https://cloud.google.com/docs/authentication/production
+
+```yaml
+out:
+  type: bigquery
+  auth_method: application_default
+```
+
 ### Table id formatting
 
 `table` and option accept [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime)

lib/embulk/output/bigquery.rb

@@ -33,9 +33,7 @@ def self.load(v)
       def self.configure(config, schema, task_count)
         task = {
           'mode'                           => config.param('mode',                           :string,  :default => 'append'),
-          'auth_method'                    => config.param('auth_method',                    :string,  :default => 'private_key'),
-          'service_account_email'          => config.param('service_account_email',          :string,  :default => nil),
-          'p12_keyfile'                    => config.param('p12_keyfile',                    :string,  :default => nil),
+          'auth_method'                    => config.param('auth_method',                    :string,  :default => 'json_key'),
           'json_keyfile'                   => config.param('json_keyfile',                  LocalFile, :default => nil),
           'project'                        => config.param('project',                        :string,  :default => nil),
           'dataset'                        => config.param('dataset',                        :string),
@@ -125,11 +123,8 @@ def self.configure(config, schema, task_count)
         end
 
         task['auth_method'] = task['auth_method'].downcase
-        unless %w[private_key json_key compute_engine application_default].include?(task['auth_method'])
-          raise ConfigError.new "`auth_method` must be one of private_key, json_key, compute_engine, application_default"
-        end
-        if task['auth_method'] == 'private_key' and task['p12_keyfile'].nil?
-          raise ConfigError.new "`p12_keyfile` is required for auth_method private_key"
+        unless %w[json_key compute_engine application_default].include?(task['auth_method'])
+          raise ConfigError.new "`auth_method` must be one of json_key, compute_engine, application_default"
         end
         if task['auth_method'] == 'json_key' and task['json_keyfile'].nil?
           raise ConfigError.new "`json_keyfile` is required for auth_method json_key"

lib/embulk/output/bigquery/google_client.rb

@@ -38,19 +38,8 @@ def client
           Embulk.logger.debug { "embulk-output-bigquery: request_options: #{client.request_options.to_h}" }
 
           case @task['auth_method']
-          when 'private_key'
-            private_key_passphrase = 'notasecret'
-            key = Google::APIClient::KeyUtils.load_from_pkcs12(@task['p12_keyfile'], private_key_passphrase)
-            auth = Signet::OAuth2::Client.new(
-              token_credential_uri: "https://accounts.google.com/o/oauth2/token",
-              audience: "https://accounts.google.com/o/oauth2/token",
-              scope: @scope,
-              issuer: @task['service_account_email'],
-              signing_key: key)
-
           when 'compute_engine'
             auth = Google::Auth::GCECredentials.new
-
           when 'json_key'
             json_key = @task['json_keyfile']
             if File.exist?(json_key)
@@ -61,10 +50,8 @@ def client
               key = StringIO.new(json_key)
               auth = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: @scope)
             end
-
           when 'application_default'
             auth = Google::Auth.get_application_default([@scope])
-
           else
             raise ConfigError, "Unknown auth method: #{@task['auth_method']}"
           end

test/test_bigquery_client.rb

@@ -61,10 +61,6 @@ def record
           def test_json_keyfile
             assert_nothing_raised { BigqueryClient.new(least_task, schema).client }
           end
-
-          def test_p12_keyfile
-            # pending
-          end
         end
 
         sub_test_case "create_dataset" do

test/test_configure.rb

@@ -18,10 +18,10 @@ def shutdown
 
       def least_config
         DataSource.new({
-          'project' => 'your_project_name',
-          'dataset' => 'your_dataset_name',
-          'table'   => 'your_table_name',
-          'p12_keyfile' => __FILE__, # fake
+          'project'      => 'your_project_name',
+          'dataset'      => 'your_dataset_name',
+          'table'        => 'your_table_name',
+          'json_keyfile' => File.join(EXAMPLE_ROOT, 'json_key.json'), # dummy
         })
       end
 
@@ -43,10 +43,8 @@ def processor_count
       def test_configure_default
         task = Bigquery.configure(least_config, schema, processor_count)
         assert_equal "append", task['mode']
-        assert_equal "private_key", task['auth_method']
-        assert_equal nil, task['service_account_email']
-        assert_equal __FILE__, task['p12_keyfile']
-        assert_equal nil, task['json_keyfile']
+        assert_equal "json_key", task['auth_method']
+        assert_equal File.read(File.join(EXAMPLE_ROOT, 'json_key.json')), task['json_keyfile']
         assert_equal "your_project_name", task['project']
         assert_equal "your_dataset_name", task['dataset']
         assert_equal nil, task['location']
@@ -132,11 +130,6 @@ def test_auth_method
         config = least_config.merge('auth_method' => 'foobar')
         assert_raise { Bigquery.configure(config, schema, processor_count) }
 
-        config = least_config.merge('auth_method' => 'private_key').tap {|h| h.delete('p12_keyfile') }
-        assert_raise { Bigquery.configure(config, schema, processor_count) }
-        config = least_config.merge('auth_method' => 'private_key', 'p12_keyfile' => 'dummy')
-        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
-
         config = least_config.merge('auth_method' => 'json_key').tap {|h| h.delete('json_keyfile') }
         assert_raise { Bigquery.configure(config, schema, processor_count) }
         config = least_config.merge('auth_method' => 'json_key', 'json_keyfile' => "#{EXAMPLE_ROOT}/json_key.json")

test/test_example.rb

@@ -8,12 +8,12 @@ class Output::Bigquery
     class TestTransaction < Test::Unit::TestCase
       def least_config
         DataSource.new({
-          'project'     => 'your_project_name',
-          'dataset'     => 'your_dataset_name',
-          'table'       => 'your_table_name',
-          'p12_keyfile' => __FILE__, # fake
-          'temp_table'  => 'temp_table', # randomly created is not good for our test
-          'path_prefix' => 'tmp/', # randomly created is not good for our test
+          'project'      => 'your_project_name',
+          'dataset'      => 'your_dataset_name',
+          'table'        => 'your_table_name',
+          'temp_table'   => 'temp_table', # randomly created is not good for our test
+          'path_prefix'  => 'tmp/', # randomly created is not good for our test
+          'json_keyfile' => File.join(EXAMPLE_ROOT, 'json_key.json'), # dummy
         })
       end