Add retry for GCE 403 concurrent operations quota errors

mmv1/third_party/terraform/transport/error_retry_predicates.go

@@ -50,6 +50,11 @@ var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
 	// already.
 	is403QuotaExceededPerMinuteError,
 
+	// GCE returns a 403 with reason CONCURRENT_OPERATIONS_QUOTA_EXCEEDED
+	// when too many operations are in flight. This is transient and clears
+	// once in-flight operations complete.
+	is403ConcurrentOperationsQuotaError,
+
 	// GCE Networks are considered unready for a brief period when certain
 	// operations are performed on them, and the scope is likely too broad to
 	// apply a mutex. If we attempt an operation w/ an unready network, retry
@@ -180,6 +185,35 @@ func is403QuotaExceededPerMinuteError(err error) (bool, string) {
 	return false, ""
 }
 
+// GCE returns a 403 when the concurrent operations quota is exceeded.
+// This is a transient error that clears once in-flight operations complete.
+// See https://github.com/hashicorp/terraform-provider-google/issues/9207
+func is403ConcurrentOperationsQuotaError(err error) (bool, string) {
+	gerr, ok := err.(*googleapi.Error)
+	if !ok {
+		return false, ""
+	}
+
+	if gerr.Code != 403 {
+		return false, ""
+	}
+
+	for _, d := range gerr.Details {
+		data, ok := d.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		dType, ok := data["@type"]
+		if ok && strings.Contains(dType.(string), "ErrorInfo") {
+			if v, ok := data["reason"]; ok && v.(string) == "CONCURRENT_OPERATIONS_QUOTA_EXCEEDED" {
+				log.Printf("[DEBUG] Dismissed an error as retryable based on error code 403 and error reason 'CONCURRENT_OPERATIONS_QUOTA_EXCEEDED': %s", err)
+				return true, "Concurrent operations quota exceeded, retrying"
+			}
+		}
+	}
+	return false, ""
+}
+
 // We've encountered a few common fingerprint-related strings; if this is one of
 // them, we're confident this is an error due to fingerprints.
 var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}

mmv1/third_party/terraform/transport/error_retry_predicates_test.go

@@ -204,6 +204,57 @@ func TestFirestoreIndex409_retryUnderlyingDataChanged(t *testing.T) {
 	}
 }
 
+func TestIs403ConcurrentOperationsQuotaError_retryable(t *testing.T) {
+	err := googleapi.Error{
+		Code: 403,
+		Body: "Rate Limit Exceeded",
+		Details: []interface{}{
+			map[string]interface{}{
+				"@type":  "type.googleapis.com/google.rpc.ErrorInfo",
+				"reason": "CONCURRENT_OPERATIONS_QUOTA_EXCEEDED",
+			},
+		},
+	}
+	isRetryable, _ := is403ConcurrentOperationsQuotaError(&err)
+	if !isRetryable {
+		t.Errorf("Error not detected as retryable")
+	}
+}
+
+func TestIs403ConcurrentOperationsQuotaError_wrongReason(t *testing.T) {
+	err := googleapi.Error{
+		Code: 403,
+		Body: "Rate Limit Exceeded",
+		Details: []interface{}{
+			map[string]interface{}{
+				"@type":  "type.googleapis.com/google.rpc.ErrorInfo",
+				"reason": "RATE_LIMIT_EXCEEDED",
+			},
+		},
+	}
+	isRetryable, _ := is403ConcurrentOperationsQuotaError(&err)
+	if isRetryable {
+		t.Errorf("Error incorrectly detected as retryable")
+	}
+}
+
+func TestIs403ConcurrentOperationsQuotaError_wrongCode(t *testing.T) {
+	err := googleapi.Error{
+		Code: 400,
+		Body: "Rate Limit Exceeded",
+		Details: []interface{}{
+			map[string]interface{}{
+				"@type":  "type.googleapis.com/google.rpc.ErrorInfo",
+				"reason": "CONCURRENT_OPERATIONS_QUOTA_EXCEEDED",
+			},
+		},
+	}
+	isRetryable, _ := is403ConcurrentOperationsQuotaError(&err)
+	if isRetryable {
+		t.Errorf("Error incorrectly detected as retryable")
+	}
+}
+
 func TestExternalIpServiceNotActive(t *testing.T) {
 	err := googleapi.Error{
 		Code: 400,