Skip to content

Commit ba39232

Browse files
author
Steven Nemetz
committed
Move common routines to a library. Switch to using AWS Secrets Manager
1 parent b13d957 commit ba39232

File tree

3 files changed

+124
-101
lines changed

3 files changed

+124
-101
lines changed

‎src/slack/config.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
{
22
"aws_region": "us-west-2",
3-
"datadog_api_key": "",
4-
"datadog_app_key": "",
53
"environment": "",
64
"log_level": "DEBUG",
75
"s3_bucket_parts": "wiser-one-ci",
8-
"secret_name": "slack_webhook_datadog",
96
"secret_endpoint_url": "https://secretsmanager.us-west-2.amazonaws.com",
7+
"secrets": {
8+
"datadog_api": "datadog_api_keys",
9+
"slack_webhook": "slack_webhook_datadog"
10+
},
1011
"slack_token": "",
1112
"path_parts": "datadog/integration/slack",
1213
"parts_pattern": "channel-*.json"

‎src/slack/datadog-slack-integration.py‎

Lines changed: 7 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import boto3
33
import fnmatch
44
import json
5+
import lambda_lib
56
import logging
67
import os
78
import requests
@@ -16,84 +17,6 @@
1617
logger.setLevel(logging.DEBUG)
1718
states = {}
1819

19-
def get_config():
20-
# Allow different config file to be specified
21-
if 'LAMBDA_CONFIG' in os.environ and len(os.getenv('LAMBDA_CONFIG')) > 0:
22-
config_file = os.getenv('LAMBDA_CONFIG')
23-
if config_file.find("s3://") == 0:
24-
# read from S3 or copy default to s3
25-
s3_bucket = config_file[5:len(config_file)].split("/")[0]
26-
sep = "/"
27-
s3_object = sep.join(config_file[5:len(config_file)].split("/")[1:])
28-
s3 = boto3.client('s3')
29-
try:
30-
config_result = s3.get_object(Bucket=s3_bucket, Key=s3_object)
31-
logger.debug("S3 get config: {}".format(config_result))
32-
config_raw = config_result["Body"].read()
33-
config = json.loads(config_raw)
34-
except ClientError as e:
35-
logger.warn("Config not found in S3: {}".format(e))
36-
logger.warn("Copying default config to S3")
37-
with open('config.json') as f:
38-
config = json.load(f)
39-
s3.put_object(Bucket=s3_bucket, Key=s3_object, Body=json.dumps(config, indent=2))
40-
else:
41-
logger.debug("Reading config file: {}".format(config_file))
42-
with open(config_file) as f:
43-
config = json.load(f)
44-
else:
45-
logger.debug("Reading config file: config.json")
46-
with open('config.json') as f:
47-
config = json.load(f)
48-
49-
logger.debug("Config before overrides: {}".format(json.dumps(config, indent=2)))
50-
# Replace config data with environment data if it exists
51-
# Environment variables are keys uppercased
52-
for key in config.keys():
53-
if key.upper() in os.environ:
54-
value = os.getenv(key.upper())
55-
try:
56-
value_json = json.loads(value)
57-
logger.debug("Got json: {}".format(json.dumps(value_json)))
58-
if len(value_json.keys()) > 0:
59-
config[key] = value_json
60-
except:
61-
if len(value) > 0:
62-
config[key] = value
63-
logger.debug("Config after overrides: {}".format(json.dumps(config, indent=2)))
64-
return config
65-
66-
def get_secret(secret_name, endpoint_url, region_name):
67-
session = boto3.session.Session()
68-
client = session.client(
69-
service_name='secretsmanager',
70-
region_name=region_name,
71-
endpoint_url=endpoint_url
72-
)
73-
74-
try:
75-
get_secret_value_response = client.get_secret_value(
76-
SecretId=secret_name
77-
)
78-
except ClientError as e:
79-
if e.response['Error']['Code'] == 'ResourceNotFoundException':
80-
print("The requested secret " + secret_name + " was not found")
81-
elif e.response['Error']['Code'] == 'InvalidRequestException':
82-
print("The request was invalid due to:", e)
83-
elif e.response['Error']['Code'] == 'InvalidParameterException':
84-
print("The request had invalid params:", e)
85-
else:
86-
# Decrypted secret using the associated KMS CMK
87-
# Depending on whether the secret was a string or binary, one of these fields will be populated
88-
if 'SecretString' in get_secret_value_response:
89-
secret = get_secret_value_response['SecretString']
90-
#print(json.dumps(json.loads(secret), indent=2))
91-
#print(secret)
92-
return secret
93-
else:
94-
binary_secret_data = get_secret_value_response['SecretBinary']
95-
return binary_secret_data
96-
9720
def get_slack_channels(s3_bucket, prefix, pattern):
9821
# Return: list of channel names
9922
channels = []
@@ -296,31 +219,15 @@ def get_slack_hooks(url):
296219
return hooks
297220

298221
def write_datadog_slack(config, channels, service_hooks):
299-
# Datadog: url, api_key, app_key
300-
# POST: Create integration: Does add, not delete or update. Can have duplicate entries
301-
# PUT: Create/Update integration: updates, deletes
302222
data = {}
303223
data["channels"] = channels
304224
data["service_hooks"] = service_hooks
305225
#print(json.dumps(data, indent=2))
306-
url_base = "https://api.datadoghq.com/api/v1/integration/slack"
307-
api = "?api_key=" + config["datadog_api_key"]
308-
app = "&application_key=" + config["datadog_app_key"]
309-
url = url_base + api + app + "&run_check=true"
310-
headers = {"Content-type": "application/json"}
311-
print("URL: {}".format(url))
312-
#result = requests.delete(url_base + api + app)
313-
#print(result)
314-
#result = requests.post(url, data=json.dumps(data), headers=headers)
315-
result = requests.put(url, data=json.dumps(data), headers=headers)
316-
print(result)
317-
if result.status_code != 204:
318-
logger.error("Datadog Slack integration update failed with status: {}".format(result.status_code))
319-
#logger.debug("HTTP response header {}".format(json.dumps(result.headers, indent=2)))
226+
lambda_lib.write_datadog_integration(config, 'slack', data)
320227

321228
def lambda_handler(event, context):
322229
global states
323-
config = get_config()
230+
config = lambda_lib.get_config()
324231

325232
# Setup logging
326233
log_level = getattr(logging, config["log_level"].upper(), None)
@@ -331,10 +238,12 @@ def lambda_handler(event, context):
331238
logger.debug("Event: {}".format(json.dumps(event, indent=2)))
332239
logger.debug("Context: {}".format(json.dumps(context, indent=2)))
333240
channels = get_slack_channels(config["s3_bucket_parts"], config["path_parts"], config["parts_pattern"])
334-
slack_webhook = get_secret(config["secret_name"], config["secret_endpoint_url"], config["aws_region"])
241+
# Secrets: DD api, DD app, slack webhook
242+
datadog_keys = json.loads(lambda_lib.get_secret(config["secrets"]["datadog_api"], config["secret_endpoint_url"], config["aws_region"]))
243+
slack_webhook = lambda_lib.get_secret(config["secrets"]["slack_webhook"], config["secret_endpoint_url"], config["aws_region"])
335244
slack_hooks = get_slack_hooks(slack_webhook)
336245
#update_slack_channels(channels)
337-
write_datadog_slack(config, channels, slack_hooks)
246+
write_datadog_slack(datadog_keys, channels, slack_hooks)
338247

339248
return 'Done'
340249

‎src/slack/lambda_lib.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
from __future__ import print_function
2+
import boto3
3+
import json
4+
import logging
5+
import os
6+
import requests
7+
from botocore.exceptions import ClientError
8+
9+
10+
## No handler for root when run locally
11+
logger = logging.getLogger()
12+
logger.setLevel(logging.DEBUG)
13+
14+
def get_config():
15+
# Allow different config file to be specified
16+
if 'LAMBDA_CONFIG' in os.environ and len(os.getenv('LAMBDA_CONFIG')) > 0:
17+
config_file = os.getenv('LAMBDA_CONFIG')
18+
if config_file.find("s3://") == 0:
19+
# read from S3 or copy default to s3
20+
s3_bucket = config_file[5:len(config_file)].split("/")[0]
21+
sep = "/"
22+
s3_object = sep.join(config_file[5:len(config_file)].split("/")[1:])
23+
s3 = boto3.client('s3')
24+
try:
25+
config_result = s3.get_object(Bucket=s3_bucket, Key=s3_object)
26+
logger.debug("S3 get config: {}".format(config_result))
27+
config_raw = config_result["Body"].read()
28+
config = json.loads(config_raw)
29+
except ClientError as e:
30+
logger.warn("Config not found in S3: {}".format(e))
31+
logger.warn("Copying default config to S3")
32+
with open('config.json') as f:
33+
config = json.load(f)
34+
s3.put_object(Bucket=s3_bucket, Key=s3_object, Body=json.dumps(config, indent=2))
35+
else:
36+
logger.debug("Reading config file: {}".format(config_file))
37+
with open(config_file) as f:
38+
config = json.load(f)
39+
else:
40+
logger.debug("Reading config file: config.json")
41+
with open('config.json') as f:
42+
config = json.load(f)
43+
44+
logger.debug("Config before overrides: {}".format(json.dumps(config, indent=2)))
45+
# Replace config data with environment data if it exists
46+
# Environment variables are keys uppercased
47+
for key in config.keys():
48+
if key.upper() in os.environ:
49+
value = os.getenv(key.upper())
50+
try:
51+
value_json = json.loads(value)
52+
logger.debug("Got json: {}".format(json.dumps(value_json)))
53+
if len(value_json.keys()) > 0:
54+
config[key] = value_json
55+
except:
56+
if len(value) > 0:
57+
config[key] = value
58+
logger.debug("Config after overrides: {}".format(json.dumps(config, indent=2)))
59+
return config
60+
61+
def get_secret(secret_name, endpoint_url, region_name):
62+
# TODO: add json support
63+
session = boto3.session.Session()
64+
client = session.client(
65+
service_name='secretsmanager',
66+
region_name=region_name,
67+
endpoint_url=endpoint_url
68+
)
69+
70+
print("Lookup secret: {}".format(secret_name))
71+
try:
72+
get_secret_value_response = client.get_secret_value(
73+
SecretId=secret_name
74+
)
75+
except ClientError as e:
76+
if e.response['Error']['Code'] == 'ResourceNotFoundException':
77+
print("The requested secret " + secret_name + " was not found")
78+
elif e.response['Error']['Code'] == 'InvalidRequestException':
79+
# Secret name not exists?
80+
print("The request was invalid due to:", e)
81+
elif e.response['Error']['Code'] == 'InvalidParameterException':
82+
print("The request had invalid params:", e)
83+
else:
84+
# Decrypted secret using the associated KMS CMK
85+
# Depending on whether the secret was a string or binary, one of these fields will be populated
86+
if 'SecretString' in get_secret_value_response:
87+
secret = get_secret_value_response['SecretString']
88+
#print(json.dumps(json.loads(secret), indent=2))
89+
#print(secret)
90+
return secret
91+
else:
92+
binary_secret_data = get_secret_value_response['SecretBinary']
93+
return binary_secret_data
94+
95+
def write_datadog_integration(datadog_keys, integration, payload):
96+
# Datadog: url, api_key, app_key
97+
# POST: Create integration: Does add, not delete or update. Can have duplicate entries
98+
# PUT: Create/Update integration: updates, deletes
99+
#print(json.dumps(payload, indent=2))
100+
url_base = "https://api.datadoghq.com/api/v1/integration/" + integration
101+
api = "?api_key=" + datadog_keys["api_key"]
102+
app = "&application_key=" + datadog_keys["app_key"]
103+
url = url_base + api + app + "&run_check=true"
104+
headers = {"Content-type": "application/json"}
105+
print("URL: {}".format(url))
106+
#result = requests.delete(url_base + api + app)
107+
#print(result)
108+
#result = requests.post(url, data=json.dumps(data), headers=headers)
109+
result = requests.put(url, data=json.dumps(payload), headers=headers)
110+
print(result)
111+
if result.status_code != 204:
112+
logger.error("Datadog {} integration update failed with status: {}".format(integration, result.status_code))
113+
#logger.debug("HTTP response header {}".format(json.dumps(result.headers, indent=2)))

0 commit comments

Comments
 (0)