From 521bb6b24a85df232bb767844350e65efc3dd8bd Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 13 Feb 2013 11:29:59 -0800 Subject: [PATCH 1/5] add td-agent cookbook --- cookbooks/td-agent/README.rdoc | 44 +++++++++++++ cookbooks/td-agent/attributes/default.rb | 1 + cookbooks/td-agent/metadata.rb | 19 ++++++ cookbooks/td-agent/recipes/default.rb | 62 +++++++++++++++++++ .../templates/default/td-agent.conf.erb | 22 +++++++ 5 files changed, 148 insertions(+) create mode 100644 cookbooks/td-agent/README.rdoc create mode 100644 cookbooks/td-agent/attributes/default.rb create mode 100644 cookbooks/td-agent/metadata.rb create mode 100644 cookbooks/td-agent/recipes/default.rb create mode 100644 cookbooks/td-agent/templates/default/td-agent.conf.erb diff --git a/cookbooks/td-agent/README.rdoc b/cookbooks/td-agent/README.rdoc new file mode 100644 index 00000000..2236f187 --- /dev/null +++ b/cookbooks/td-agent/README.rdoc @@ -0,0 +1,44 @@ += DESCRIPTION: + +{Opscode Chef}[http://www.opscode.com/chef/] cookbook for td-agent (Treasure Data Agent). The release log of td-agent is available {here}[http://help.treasure-data.com/kb/installing-td-agent-daemon/changelog-of-td-agent]. + +NOTE: td-agent is open-sourced as {Fluentd project}[http://github.com/fluent/]. If you want to use stable version of Fluentd, using this cookbook is recommended. + += INSTALLATION: + +The {knife-github-cookbooks}[https://github.com/websterclay/knife-github-cookbooks] gem is a plugin for knife that supports installing cookbooks directly from a GitHub repository. To install with this plugin, please follow these steps: + + $ gem install knife-github-cookbooks + $ cd chef-repo + $ knife cookbook github install treasure-data/chef-td-agent + += REQUIREMENTS: + +This cookbook has these external dependencies. + +- apt cookbook +- yum cookbook + += ATTRIBUTES: + +API Key, and the Secret Key are required. + +- node[:td_agent][:api_key] (required) + += USAGE: + +This is an example role file. + + name "base" + description "base server role." + run_list( + "recipe[apt]", + "recipe[yum]", + "recipe[td-agent]", + ) + override_attributes( + # for td-agent + :td_agent => { + :api_key => 'foo_bar_buz', + } + ) \ No newline at end of file diff --git a/cookbooks/td-agent/attributes/default.rb b/cookbooks/td-agent/attributes/default.rb new file mode 100644 index 00000000..a78bf529 --- /dev/null +++ b/cookbooks/td-agent/attributes/default.rb @@ -0,0 +1 @@ +default[:td_agent][:api_key] = '' diff --git a/cookbooks/td-agent/metadata.rb b/cookbooks/td-agent/metadata.rb new file mode 100644 index 00000000..88e02324 --- /dev/null +++ b/cookbooks/td-agent/metadata.rb @@ -0,0 +1,19 @@ +maintainer "Treasure Data, Inc." +maintainer_email "k@treasure-data.com" +license "All rights reserved" +description "Installs/Configures td-agent" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc')) +version "0.0.1" +recipe "td-agent", "td-agent configuration" + +%w{redhat centos debian ubuntu}.each do |os| + supports os +end + +depends 'apt' +depends 'yum' + +attribute "td_agent/api_key", + :display_name => "Treasure Data ApiKey", + :description => "ApiKey for Treasure Data Service", + :default => '' diff --git a/cookbooks/td-agent/recipes/default.rb b/cookbooks/td-agent/recipes/default.rb new file mode 100644 index 00000000..4988616f --- /dev/null +++ b/cookbooks/td-agent/recipes/default.rb @@ -0,0 +1,62 @@ +# +# Cookbook Name:: td-agent +# Recipe:: default +# +# Copyright 2011, Treasure Data, Inc. +# + +group 'td-agent' do + group_name 'td-agent' + gid 403 + action [:create] +end + +user 'td-agent' do + comment 'td-agent' + uid 403 + group 'td-agent' + home '/var/run/td-agent' + shell '/bin/false' + password nil + supports :manage_home => true + action [:create, :manage] +end + +directory '/etc/td-agent/' do + owner 'td-agent' + group 'td-agent' + mode '0755' + action :create +end + +case node['platform'] +when "ubuntu" + dist = node['lsb']['codename'] + source = (dist == 'precise') ? "http://packages.treasure-data.com/precise/" : "http://packages.treasure-data.com/debian/" + apt_repository "treasure-data" do + uri source + distribution dist + components ["contrib"] + action :add + end +when "centos", "redhat" + yum_repository "treasure-data" do + url "http://packages.treasure-data.com/redhat/$basearch" + action :add + end +end + +template "/etc/td-agent/td-agent.conf" do + mode "0644" + source "td-agent.conf.erb" +end + +package "td-agent" do + options "-f --force-yes" + action :upgrade +end + +service "td-agent" do + action [ :enable, :start ] + subscribes :restart, resources(:template => "/etc/td-agent/td-agent.conf") +end diff --git a/cookbooks/td-agent/templates/default/td-agent.conf.erb b/cookbooks/td-agent/templates/default/td-agent.conf.erb new file mode 100644 index 00000000..3baf693f --- /dev/null +++ b/cookbooks/td-agent/templates/default/td-agent.conf.erb @@ -0,0 +1,22 @@ +# Source descriptions +## built-in TCP input + + type tcp + port 24224 + + +## HTTP input + + type http + port 8888 + + +# Match descriptions +## Treasure Data output +## match events whose tag is td.DATABASE.TABLE + + type tdlog + apikey <%= node['td_agent']['api_key'] %> + buffer_type file + buffer_path /var/log/td-agent/buffer/td + From 5de6074d15b258645f0d1eebc9a3f394111fd0e0 Mon Sep 17 00:00:00 2001 From: Ines Sombra Date: Tue, 19 Feb 2013 14:11:05 -0800 Subject: [PATCH 2/5] Requires TD recipe --- cookbooks/main/recipes/default.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cookbooks/main/recipes/default.rb b/cookbooks/main/recipes/default.rb index 0d7c6429..b4485478 100644 --- a/cookbooks/main/recipes/default.rb +++ b/cookbooks/main/recipes/default.rb @@ -138,3 +138,6 @@ # postgresql9_pg_buffercache "postgres" # postgresql9_pg_freespacemap "postgres" #end + +# Install Treasure Data Agent +require recipe "td-agent" From 4a9b97e9076175c3f0175ceb0e9243a95598287b Mon Sep 17 00:00:00 2001 From: Ines Sombra Date: Tue, 19 Feb 2013 14:21:03 -0800 Subject: [PATCH 3/5] fixing api_key definition --- cookbooks/td-agent/attributes/default.rb | 2 +- cookbooks/td-agent/templates/default/td-agent.conf.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbooks/td-agent/attributes/default.rb b/cookbooks/td-agent/attributes/default.rb index a78bf529..9ab49d84 100644 --- a/cookbooks/td-agent/attributes/default.rb +++ b/cookbooks/td-agent/attributes/default.rb @@ -1 +1 @@ -default[:td_agent][:api_key] = '' +td_agent_api_key('') diff --git a/cookbooks/td-agent/templates/default/td-agent.conf.erb b/cookbooks/td-agent/templates/default/td-agent.conf.erb index 3baf693f..0cca8897 100644 --- a/cookbooks/td-agent/templates/default/td-agent.conf.erb +++ b/cookbooks/td-agent/templates/default/td-agent.conf.erb @@ -16,7 +16,7 @@ ## match events whose tag is td.DATABASE.TABLE type tdlog - apikey <%= node['td_agent']['api_key'] %> + apikey <%= @td_agent_api_key%> buffer_type file buffer_path /var/log/td-agent/buffer/td From 72ac135fc25e692af098a6c2c13e9f213b56dc76 Mon Sep 17 00:00:00 2001 From: Ines Sombra Date: Tue, 19 Feb 2013 14:23:17 -0800 Subject: [PATCH 4/5] adding test key before initial environment push --- cookbooks/td-agent/attributes/default.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbooks/td-agent/attributes/default.rb b/cookbooks/td-agent/attributes/default.rb index 9ab49d84..bcd6dee3 100644 --- a/cookbooks/td-agent/attributes/default.rb +++ b/cookbooks/td-agent/attributes/default.rb @@ -1 +1,2 @@ -td_agent_api_key('') +# my testing key - will be deleted before merging the pull request +td_agent_api_key('1f157707e42fe82016b32b01aa884bfc8a3b9054') From b7e0b2addfb1704ec13afa84288f4c3b55608cd7 Mon Sep 17 00:00:00 2001 From: Ines Sombra Date: Tue, 19 Feb 2013 15:06:32 -0800 Subject: [PATCH 5/5] wip --- cookbooks/main/recipes/default.rb | 2 +- cookbooks/td-agent/attributes/default.rb | 2 +- cookbooks/td-agent/recipes/default.rb | 90 +++++++++++++----------- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/cookbooks/main/recipes/default.rb b/cookbooks/main/recipes/default.rb index b4485478..b1ada73a 100644 --- a/cookbooks/main/recipes/default.rb +++ b/cookbooks/main/recipes/default.rb @@ -140,4 +140,4 @@ #end # Install Treasure Data Agent -require recipe "td-agent" +require_recipe "td-agent" diff --git a/cookbooks/td-agent/attributes/default.rb b/cookbooks/td-agent/attributes/default.rb index bcd6dee3..4ee4433e 100644 --- a/cookbooks/td-agent/attributes/default.rb +++ b/cookbooks/td-agent/attributes/default.rb @@ -1,2 +1,2 @@ # my testing key - will be deleted before merging the pull request -td_agent_api_key('1f157707e42fe82016b32b01aa884bfc8a3b9054') +td_agent_api_key("1f157707e42fe82016b32b01aa884bfc8a3b9054") diff --git a/cookbooks/td-agent/recipes/default.rb b/cookbooks/td-agent/recipes/default.rb index 4988616f..dcad66d2 100644 --- a/cookbooks/td-agent/recipes/default.rb +++ b/cookbooks/td-agent/recipes/default.rb @@ -5,58 +5,64 @@ # Copyright 2011, Treasure Data, Inc. # -group 'td-agent' do - group_name 'td-agent' - gid 403 - action [:create] -end +td_api_key = node[:td_agent_api_key] -user 'td-agent' do - comment 'td-agent' - uid 403 - group 'td-agent' - home '/var/run/td-agent' - shell '/bin/false' - password nil - supports :manage_home => true - action [:create, :manage] +ey_cloud_report "Installing Treasure Data Agent" do + message "installing Treasure Data Agent" end -directory '/etc/td-agent/' do - owner 'td-agent' - group 'td-agent' - mode '0755' +#group 'td-agent' do + #group_name 'td-agent' + #gid 403 + #action :create +#end + +#user 'td-agent' do + #comment 'td-agent' + #uid 403 + #group 'td-agent' + #home '/var/run/td-agent' + #shell '/bin/false' + #password nil + #supports :manage_home => true + #action :create, :manage +#end + +directory "/etc/td-agent/" do + owner node[:owner_name] + group node[:owner_name] + mode 0755 action :create end -case node['platform'] -when "ubuntu" - dist = node['lsb']['codename'] - source = (dist == 'precise') ? "http://packages.treasure-data.com/precise/" : "http://packages.treasure-data.com/debian/" - apt_repository "treasure-data" do - uri source - distribution dist - components ["contrib"] - action :add - end -when "centos", "redhat" - yum_repository "treasure-data" do - url "http://packages.treasure-data.com/redhat/$basearch" - action :add - end -end +#case node['platform'] +#when "ubuntu" + #dist = node['lsb']['codename'] + #source = (dist == 'precise') ? "http://packages.treasure-data.com/precise/" : "http://packages.treasure-data.com/debian/" + #apt_repository "treasure-data" do + #uri source + #distribution dist + #components ["contrib"] + #action :add + #end +#when "centos", "redhat" + #yum_repository "treasure-data" do + #url "http://packages.treasure-data.com/redhat/$basearch" + #action :add + #end +#end template "/etc/td-agent/td-agent.conf" do mode "0644" source "td-agent.conf.erb" end -package "td-agent" do - options "-f --force-yes" - action :upgrade -end +#package "td-agent" do + #options "-f --force-yes" + #action :upgrade +#end -service "td-agent" do - action [ :enable, :start ] - subscribes :restart, resources(:template => "/etc/td-agent/td-agent.conf") -end +#service "td-agent" do + #action [ :enable, :start ] + #subscribes :restart, resources(:template => "/etc/td-agent/td-agent.conf") +#end