fix(BREV-2099): Handle Shadeform 409 Errors (brevdev#57)

* Shadeform insufficient capacity (brevdev#53)

* fix: add insufficent capacity handling for shadeform

* fix: uncomment shadeform tests

* fix(BREV-2099): Handle shadeform 409 errors

* wrap and trace, use errors is in test

---------

Co-authored-by: Ronald Ding <ronaldhding@gmail.com>

Author: drewmalin

v1/providers/shadeform/instance.go

@@ -2,8 +2,10 @@ package v1
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"strings"
 
 	"github.com/alecthomas/units"
@@ -19,8 +21,14 @@ const (
 	cloudCredRefIDTagName = "cloudCredRefID" //nolint:gosec // not a secret
 	instanceNameFormat    = "%v_%v"
 	instanceNameSeparator = "_"
+	outOfStockErrorCode   = "OUT_OF_STOCK"
 )
 
+type DefaultErrorResponse struct {
+	ErrorCode string `json:"error_code"`
+	Error     string `json:"error"`
+}
+
 func (c *ShadeformClient) CreateInstance(ctx context.Context, attrs v1.CreateInstanceAttrs) (*v1.Instance, error) { //nolint:gocyclo,funlen // ok
 	authCtx := c.makeAuthContext(ctx)
 
@@ -110,10 +118,8 @@ func (c *ShadeformClient) CreateInstance(ctx context.Context, attrs v1.CreateIns
 		defer func() { _ = httpResp.Body.Close() }()
 	}
 	if err != nil {
-		httpMessage, _ := io.ReadAll(httpResp.Body)
-		return nil, errors.WrapAndTrace(fmt.Errorf("failed to create instance: %w, %s", err, string(httpMessage)))
+		return nil, errors.WrapAndTrace(fmt.Errorf("failed to create instance: %w", handleShadeformAPIErrorResponse(httpResp)))
 	}
-
 	if resp == nil {
 		return nil, errors.WrapAndTrace(fmt.Errorf("no instance returned from create request"))
 	}
@@ -128,6 +134,36 @@ func (c *ShadeformClient) CreateInstance(ctx context.Context, attrs v1.CreateIns
 	return createdInstance, nil
 }
 
+func handleShadeformAPIErrorResponse(httpResp *http.Response) error {
+	// Read the details of the API response
+	httpStatusCode := httpResp.StatusCode
+	httpMessageBytes, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return errors.WrapAndTrace(fmt.Errorf("failed to read shadeform API response body: %w", err))
+	}
+
+	// Most well-structured errors are 409, so handle these as a special case
+	if httpStatusCode == http.StatusConflict {
+		// Unmarshal the response body into a Shadeform DefaultErrorResponse
+		var shadeformErrorResponse DefaultErrorResponse
+		err = json.Unmarshal(httpMessageBytes, &shadeformErrorResponse)
+		if err != nil {
+			return errors.WrapAndTrace(fmt.Errorf("failed to unmarshal shadeform API response body: %w", err))
+		}
+
+		// Handle Shadeform specific errors and attempt to translate them into Brev errors
+		if shadeformErrorResponse.ErrorCode == outOfStockErrorCode {
+			return v1.ErrInsufficientResources
+		} else {
+			// For all other error codes, return the error object from the API response
+			return errors.WrapAndTrace(fmt.Errorf("shadeform API error: error code: %v, error: %v", shadeformErrorResponse.ErrorCode, shadeformErrorResponse.Error))
+		}
+	}
+
+	// For all other HTTP status codes, return the status code and body
+	return errors.WrapAndTrace(fmt.Errorf("shadeform HTTP error: [%d], %s", httpStatusCode, string(httpMessageBytes)))
+}
+
 func (c *ShadeformClient) getInstanceNameForShadeform(refID string, providedName string) string {
 	return fmt.Sprintf(instanceNameFormat, refID, providedName)
 }

v1/providers/shadeform/validation_test.go

@@ -2,14 +2,19 @@ package v1
 
 import (
 	"context"
+	"errors"
+	"io"
+	"net/http"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
-	"github.com/brevdev/cloud/internal/ssh"
 	"github.com/brevdev/cloud/internal/validation"
-	v1 "github.com/brevdev/cloud/v1"
 	openapi "github.com/brevdev/cloud/v1/providers/shadeform/gen/shadeform"
+
+	"github.com/brevdev/cloud/internal/ssh"
+	v1 "github.com/brevdev/cloud/v1"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
@@ -118,6 +123,90 @@ func TestInstanceTypeFilter(t *testing.T) {
 	})
 }
 
+func TestHandleShadeformAPIErrorResponse(t *testing.T) {
+	tests := []struct {
+		name          string
+		httpResp      *http.Response
+		expectedError error
+	}{
+		{
+			name:          "OutOfStockError",
+			httpResp:      &http.Response{StatusCode: http.StatusConflict, Body: io.NopCloser(strings.NewReader(`{"error_code": "OUT_OF_STOCK", "error": "Out of stock"}`))},
+			expectedError: v1.ErrInsufficientResources,
+		},
+		{
+			name:          "OtherShadeformError",
+			httpResp:      &http.Response{StatusCode: http.StatusConflict, Body: io.NopCloser(strings.NewReader(`{"error_code": "INTERNAL_SERVER_ERROR", "error": "Internal server error"}`))},
+			expectedError: errors.New(`shadeform API error: error code: INTERNAL_SERVER_ERROR, error: Internal server error`),
+		},
+		{
+			name:          "OtherHTTPError",
+			httpResp:      &http.Response{StatusCode: http.StatusInternalServerError, Body: io.NopCloser(strings.NewReader("error"))},
+			expectedError: errors.New("shadeform HTTP error: [500], error"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := handleShadeformAPIErrorResponse(tt.httpResp)
+			require.Contains(t, err.Error(), tt.expectedError.Error())
+		})
+	}
+}
+
+func TestOutOfStockError(t *testing.T) {
+	checkSkip(t)
+	apiKey := getAPIKey()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	t.Cleanup(cancel)
+
+	client := NewShadeformClient("validation-test", apiKey)
+	client.WithConfiguration(Configuration{})
+
+	_, err := client.CreateInstance(ctx, v1.CreateInstanceAttrs{
+		RefID:        uuid.New().String(),
+		InstanceType: "datacrunch_H200x8",
+		Location:     "abc123", // Put a region that is not valid
+		PublicKey:    ssh.GetTestPublicKey(),
+		Name:         "test_name",
+		FirewallRules: v1.FirewallRules{
+			EgressRules: []v1.FirewallRule{
+				{
+					ID:       "test-rule1",
+					FromPort: 80,
+					ToPort:   8080,
+					IPRanges: []string{"127.0.0.1", "10.0.0.0/24"},
+				},
+				{
+					ID:       "test-rule2",
+					FromPort: 5432,
+					ToPort:   5432,
+					IPRanges: []string{"127.0.0.1", "10.0.0.0/24"},
+				},
+			},
+			IngressRules: []v1.FirewallRule{
+				{
+					ID:       "test-rule3",
+					FromPort: 80,
+					ToPort:   8080,
+					IPRanges: []string{"127.0.0.1", "10.0.0.0/24"},
+				},
+				{
+					ID:       "test-rule4",
+					FromPort: 5432,
+					ToPort:   5432,
+					IPRanges: []string{"127.0.0.1", "10.0.0.0/24"},
+				},
+			},
+		},
+	})
+	if err == nil {
+		t.Fatalf("ValidateCreateInstance failed: Should have resulted in an insufficientResourcesError")
+	}
+	require.True(t, errors.Is(err, v1.ErrInsufficientResources), "Error must be ErrInsufficientResources")
+}
+
 func checkSkip(t *testing.T) {
 	apiKey := getAPIKey()
 	isValidationTest := os.Getenv("VALIDATION_TEST")