Skip to content

Update: Add Retryer mechanism to existing auto scaling group operatio… #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
claudiumacovei wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update: Add Retryer mechanism to existing auto scaling group operatio… #23

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 56 additions & 5 deletions ...re/src/main/java/com/adobe/aam/shredder/core/aws/servergroup/AwsAutoScaleGroupHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,56 @@
package com.adobe.aam.shredder.core.aws.servergroup;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonWebServiceRequest;
import com.amazonaws.services.autoscaling.AmazonAutoScaling;
import com.amazonaws.services.autoscaling.model.*;
import com.github.rholder.retry.Attempt;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.RetryListener;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

public class AwsAutoScaleGroupHelper implements AutoScaleGroupHelper {

private static final Logger LOG = LoggerFactory.getLogger(AwsAutoScaleGroupHelper.class);

private final Retryer<DescribeAutoScalingInstancesResult> describeAutoScalingInstancesRetryer = RetryerBuilder.<DescribeAutoScalingInstancesResult>newBuilder()
.retryIfException(this::shouldRetryOnException)
.withWaitStrategy(WaitStrategies.randomWait(2, TimeUnit.MINUTES))
.withStopStrategy(StopStrategies.stopAfterAttempt(7))
.withRetryListener(new RetryListener() {
@Override
public <V> void onRetry(Attempt<V> attempt) {
if (attempt.hasException()) {
LOG.warn("Failed to send lifecycle event to AWS. {}", attempt.getExceptionCause().getMessage());
}
}
})
.build();

private final Retryer<DescribeLifecycleHooksResult> describeLifecycleHooksRetryer = RetryerBuilder.<DescribeLifecycleHooksResult>newBuilder()
.retryIfException(this::shouldRetryOnException)
.withWaitStrategy(WaitStrategies.randomWait(2, TimeUnit.MINUTES))
.withStopStrategy(StopStrategies.stopAfterAttempt(7))
.withRetryListener(new RetryListener() {
@Override
public <V> void onRetry(Attempt<V> attempt) {
if (attempt.hasException()) {
LOG.warn("Failed to send lifecycle event to AWS. {}", attempt.getExceptionCause().getMessage());
}
}
})
.build();

private AmazonAutoScaling asg;

public AwsAutoScaleGroupHelper(AmazonAutoScaling asg) {
Expand All @@ -37,39 +74,47 @@ public AwsAutoScaleGroupHelper(AmazonAutoScaling asg) {
public Optional<AutoScalingInstanceDetails> getCurrentAutoScalingGroup(String instanceId) {
try {
DescribeAutoScalingInstancesRequest request = new DescribeAutoScalingInstancesRequest().withInstanceIds(instanceId);
DescribeAutoScalingInstancesResult describeResult = asg.describeAutoScalingInstances(request);
DescribeAutoScalingInstancesResult describeResult = describeAutoScalingInstancesRetryer.call(getCurrentAutoScalingGroupCommand(request));

List<AutoScalingInstanceDetails> asgs = describeResult.getAutoScalingInstances();
if (asgs.isEmpty()) {
return Optional.empty();
}

return Optional.ofNullable(asgs.iterator().next());
} catch (AmazonClientException e) {
} catch (AmazonClientException | ExecutionException | RetryException e) {
LOG.error("Unable to fetch current AutoScaleGroup for instance: {} {}", instanceId, e);
return Optional.empty();
}
}

private Callable<DescribeAutoScalingInstancesResult> getCurrentAutoScalingGroupCommand(DescribeAutoScalingInstancesRequest request) {
return () -> asg.describeAutoScalingInstances(request);
}

@Override
public Optional<String> getLifecycleHookName(String autoScaleGroupName, String transitionType) {
try {
DescribeLifecycleHooksRequest request = new DescribeLifecycleHooksRequest()
.withAutoScalingGroupName(autoScaleGroupName);
DescribeLifecycleHooksResult hooks = asg.describeLifecycleHooks(request);
DescribeLifecycleHooksResult hooks = describeLifecycleHooksRetryer.call(getLifecycleHookNameCommand(request));
for (LifecycleHook hook : hooks.getLifecycleHooks()) {
if (transitionType.equals(hook.getLifecycleTransition())) {
return Optional.ofNullable(hook.getLifecycleHookName());
}
}

return Optional.empty();
} catch (AmazonClientException e) {
} catch (AmazonClientException | ExecutionException | RetryException e) {
LOG.error("Unable to fetch current Lifecycle Hook {}", transitionType, e);
return Optional.empty();
}
}

private Callable<DescribeLifecycleHooksResult> getLifecycleHookNameCommand(DescribeLifecycleHooksRequest request) {
return () -> asg.describeLifecycleHooks(request);
}

@Override
public void recordLifecycleActionHeartbeat(RecordLifecycleActionHeartbeatRequest request) {
asg.recordLifecycleActionHeartbeat(request);
Expand All @@ -79,4 +124,10 @@ public void recordLifecycleActionHeartbeat(RecordLifecycleActionHeartbeatRequest
public void completeLifecycleAction(CompleteLifecycleActionRequest request) {
asg.completeLifecycleAction(request);
}

private boolean shouldRetryOnException(Throwable throwable) {
return throwable != null
&& throwable.getMessage() != null
&& throwable.getMessage().contains("Rate exceeded");
}
}