From 169df85b9086d2ea33602aed317b385830466c69 Mon Sep 17 00:00:00 2001 From: Petrisor-Claudiu Macovei Date: Fri, 28 Jul 2023 11:34:33 +0300 Subject: [PATCH] Update: Add Retryer mechanism to existing auto scaling group operations (refs #215) --- .../servergroup/AwsAutoScaleGroupHelper.java | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/shredder-core/src/main/java/com/adobe/aam/shredder/core/aws/servergroup/AwsAutoScaleGroupHelper.java b/shredder-core/src/main/java/com/adobe/aam/shredder/core/aws/servergroup/AwsAutoScaleGroupHelper.java index 43f6125..541aea2 100644 --- a/shredder-core/src/main/java/com/adobe/aam/shredder/core/aws/servergroup/AwsAutoScaleGroupHelper.java +++ b/shredder-core/src/main/java/com/adobe/aam/shredder/core/aws/servergroup/AwsAutoScaleGroupHelper.java @@ -13,19 +13,56 @@ package com.adobe.aam.shredder.core.aws.servergroup; import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.services.autoscaling.AmazonAutoScaling; import com.amazonaws.services.autoscaling.model.*; +import com.github.rholder.retry.Attempt; +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.RetryListener; +import com.github.rholder.retry.Retryer; +import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; +import com.github.rholder.retry.WaitStrategies; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; public class AwsAutoScaleGroupHelper implements AutoScaleGroupHelper { private static final Logger LOG = LoggerFactory.getLogger(AwsAutoScaleGroupHelper.class); + private final Retryer describeAutoScalingInstancesRetryer = RetryerBuilder.newBuilder() + .retryIfException(this::shouldRetryOnException) + .withWaitStrategy(WaitStrategies.randomWait(2, TimeUnit.MINUTES)) + .withStopStrategy(StopStrategies.stopAfterAttempt(7)) + .withRetryListener(new RetryListener() { + @Override + public void onRetry(Attempt attempt) { + if (attempt.hasException()) { + LOG.warn("Failed to send lifecycle event to AWS. {}", attempt.getExceptionCause().getMessage()); + } + } + }) + .build(); + + private final Retryer describeLifecycleHooksRetryer = RetryerBuilder.newBuilder() + .retryIfException(this::shouldRetryOnException) + .withWaitStrategy(WaitStrategies.randomWait(2, TimeUnit.MINUTES)) + .withStopStrategy(StopStrategies.stopAfterAttempt(7)) + .withRetryListener(new RetryListener() { + @Override + public void onRetry(Attempt attempt) { + if (attempt.hasException()) { + LOG.warn("Failed to send lifecycle event to AWS. {}", attempt.getExceptionCause().getMessage()); + } + } + }) + .build(); + private AmazonAutoScaling asg; public AwsAutoScaleGroupHelper(AmazonAutoScaling asg) { @@ -37,7 +74,7 @@ public AwsAutoScaleGroupHelper(AmazonAutoScaling asg) { public Optional getCurrentAutoScalingGroup(String instanceId) { try { DescribeAutoScalingInstancesRequest request = new DescribeAutoScalingInstancesRequest().withInstanceIds(instanceId); - DescribeAutoScalingInstancesResult describeResult = asg.describeAutoScalingInstances(request); + DescribeAutoScalingInstancesResult describeResult = describeAutoScalingInstancesRetryer.call(getCurrentAutoScalingGroupCommand(request)); List asgs = describeResult.getAutoScalingInstances(); if (asgs.isEmpty()) { @@ -45,18 +82,22 @@ public Optional getCurrentAutoScalingGroup(String in } return Optional.ofNullable(asgs.iterator().next()); - } catch (AmazonClientException e) { + } catch (AmazonClientException | ExecutionException | RetryException e) { LOG.error("Unable to fetch current AutoScaleGroup for instance: {} {}", instanceId, e); return Optional.empty(); } } + private Callable getCurrentAutoScalingGroupCommand(DescribeAutoScalingInstancesRequest request) { + return () -> asg.describeAutoScalingInstances(request); + } + @Override public Optional getLifecycleHookName(String autoScaleGroupName, String transitionType) { try { DescribeLifecycleHooksRequest request = new DescribeLifecycleHooksRequest() .withAutoScalingGroupName(autoScaleGroupName); - DescribeLifecycleHooksResult hooks = asg.describeLifecycleHooks(request); + DescribeLifecycleHooksResult hooks = describeLifecycleHooksRetryer.call(getLifecycleHookNameCommand(request)); for (LifecycleHook hook : hooks.getLifecycleHooks()) { if (transitionType.equals(hook.getLifecycleTransition())) { return Optional.ofNullable(hook.getLifecycleHookName()); @@ -64,12 +105,16 @@ public Optional getLifecycleHookName(String autoScaleGroupName, String t } return Optional.empty(); - } catch (AmazonClientException e) { + } catch (AmazonClientException | ExecutionException | RetryException e) { LOG.error("Unable to fetch current Lifecycle Hook {}", transitionType, e); return Optional.empty(); } } + private Callable getLifecycleHookNameCommand(DescribeLifecycleHooksRequest request) { + return () -> asg.describeLifecycleHooks(request); + } + @Override public void recordLifecycleActionHeartbeat(RecordLifecycleActionHeartbeatRequest request) { asg.recordLifecycleActionHeartbeat(request); @@ -79,4 +124,10 @@ public void recordLifecycleActionHeartbeat(RecordLifecycleActionHeartbeatRequest public void completeLifecycleAction(CompleteLifecycleActionRequest request) { asg.completeLifecycleAction(request); } + + private boolean shouldRetryOnException(Throwable throwable) { + return throwable != null + && throwable.getMessage() != null + && throwable.getMessage().contains("Rate exceeded"); + } }