feat: added first type of pagination for findby and bug fixing

go.mod

@@ -3,12 +3,13 @@ module github.com/krateoplatformops/rest-dynamic-controller
 go 1.25.0
 
 require (
+	github.com/go-andiamo/splitter v1.2.5
 	github.com/go-logr/logr v1.4.2
 	github.com/gobuffalo/flect v1.0.3
 	github.com/google/go-cmp v0.7.0
 	github.com/krateoplatformops/plumbing v0.6.1
 	github.com/krateoplatformops/snowplow v0.0.0-20250311104630-6e215130151f
-	github.com/krateoplatformops/unstructured-runtime v0.2.7
+	github.com/krateoplatformops/unstructured-runtime v0.3.0
 	github.com/pb33f/libopenapi v0.28.0
 	github.com/stretchr/testify v1.11.1
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -25,6 +25,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-andiamo/splitter v1.2.5 h1:P3NovWMY2V14TJJSolXBvlOmGSZo3Uz+LtTl2bsV/eY=
+github.com/go-andiamo/splitter v1.2.5/go.mod h1:8WHU24t9hcMKU5FXDQb1hysSEC/GPuivIp0uKY1J8gw=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -81,8 +83,8 @@ github.com/krateoplatformops/plumbing v0.6.1 h1:UxQjxvJwj6ORso/RJePQv7PRC75tCcup
 github.com/krateoplatformops/plumbing v0.6.1/go.mod h1:mQ/sm0viyKgfR2ARzHuwCpY0rcyMKqCv8a8SOu52yYQ=
 github.com/krateoplatformops/snowplow v0.0.0-20250311104630-6e215130151f h1:Kw7J+0uCHPlWmcXeSvstQJDqwMwJ5WgETkHj4Z+UEjI=
 github.com/krateoplatformops/snowplow v0.0.0-20250311104630-6e215130151f/go.mod h1:C9UtLN04vcF+Hj79scByTjmwWvIQvAc7i8Dk3N+f6PA=
-github.com/krateoplatformops/unstructured-runtime v0.2.7 h1:HwvW/0vjLbNmLvYeh4d4EN43UQSMRS2iHacmadZbHs0=
-github.com/krateoplatformops/unstructured-runtime v0.2.7/go.mod h1:19uT87wZzRSjrfk3731Xhdt8ww7vnsXhljy4jk0cuWA=
+github.com/krateoplatformops/unstructured-runtime v0.3.0 h1:0lQDUDTViPEBx988b1JJYlVJNwbycTngsyqdaUOzUTQ=
+github.com/krateoplatformops/unstructured-runtime v0.3.0/go.mod h1:19uT87wZzRSjrfk3731Xhdt8ww7vnsXhljy4jk0cuWA=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=

internal/controllers/helpers.go

@@ -13,6 +13,7 @@ import (
 
 // isCRUpdated checks if the CR was updated by comparing the fields in the CR with the response from the API call, if existing cr fields are different from the response, it returns false
 func isCRUpdated(mg *unstructured.Unstructured, rm map[string]interface{}) (comparison.ComparisonResult, error) {
+	//log.Print("isCRUpdated - starting comparison between mg spec and rm")
 	if mg == nil {
 		return comparison.ComparisonResult{
 			IsEqual: false,

internal/controllers/restResources.go

@@ -98,26 +98,26 @@ func (h *handler) Observe(ctx context.Context, mg *unstructured.Unstructured) (c
 		return controller.ExternalObservation{}, err
 	}
 
+	// Set client properties
 	cli.Debug = meta.IsVerbose(mg)
 	cli.Resource = mg
 	cli.SetAuth = clientInfo.SetAuth
 	cli.IdentifierFields = clientInfo.Resource.Identifiers // TODO: probably redundant since we pass the resource too (`cli.Resource = mg`)
-	// Loop verbs, if `findby` action set, then set the IdentifiersMatchPolicy
+	// Loop VerbsDescription, if `findby` action set, then check if IdentifiersMatchPolicy is set, if so, set it in the client
 	for _, verb := range clientInfo.Resource.VerbsDescription {
 		if verb.Action == string(apiaction.FindBy) && verb.IdentifiersMatchPolicy != "" {
 			cli.IdentifiersMatchPolicy = verb.IdentifiersMatchPolicy
 			log.Debug("Found findby action and a IdentifiersMatchPolicy configured in RestDefinition", "policy", cli.IdentifiersMatchPolicy)
 			break
 		}
 	}
-	log.Debug("IdentifiersMatchPolicy set for client", "policy", cli.IdentifiersMatchPolicy)
 
 	var response restclient.Response
 	// Tries to tries to build the `get` action API Call, with the given statusFields and specFields values.
 	// If it is able to validate the `get` action request, returns true
 	isKnown := builder.IsResourceKnown(cli, clientInfo, mg)
 	if isKnown {
-		// Getting the external resource by its identifier (e.g GET /resources/{id}).
+		// Resource is known: getting the external resource by its identifier (e.g GET /resources/{id}).
 		apiCall, callInfo, err := builder.APICallBuilder(cli, clientInfo, apiaction.Get)
 		if apiCall == nil || callInfo == nil {
 			log.Error(fmt.Errorf("API action get not found"), "action", apiaction.Get)
@@ -153,9 +153,10 @@ func (h *handler) Observe(ctx context.Context, mg *unstructured.Unstructured) (c
 			return controller.ExternalObservation{}, err
 		}
 	} else {
-		// Resource is not known, we try to find it by its fields with a `findby` action,
+		// Resource is not known, we try to find it by its identifiers fields with a `findby` action,
 		// typically searching in the items returned by a "list" API call (e.g GET /resources).
-		// This is typically used when the resource does not have an identifier yet, e.g: before creation (first ever reconcile loop).
+		// This is typically used when the resource does not have an server-side generated identifier (e.g., ID, UUID) yet,
+		// for instance before creation (in the first ever reconcile loop).
 		apiCall, callInfo, err := builder.APICallBuilder(cli, clientInfo, apiaction.FindBy)
 		if apiCall == nil {
 			if !unstructuredtools.IsConditionSet(mg, condition.Creating()) && !unstructuredtools.IsConditionSet(mg, condition.Available()) {
@@ -215,7 +216,8 @@ func (h *handler) Observe(ctx context.Context, mg *unstructured.Unstructured) (c
 		}
 	}
 
-	// Response can be nil if the API does not return anything on get with a proper status code (204 No Content, 304 Not Modified).
+	// Response can be nil if the API does not return anything with a proper status code (204 No Content, 304 Not Modified) on an Observe call.
+	// In this case, we assume the resource is up-to-date.
 	if response.ResponseBody == nil {
 		cond := condition.Available()
 		cond.Message = "Resource is assumed to be up-to-date. Returned body is nil."
@@ -238,6 +240,7 @@ func (h *handler) Observe(ctx context.Context, mg *unstructured.Unstructured) (c
 			ResourceUpToDate: true,
 		}, nil
 	}
+	// If we have a response body, we try to populate status fields and check if the resource is up-to-date by comparing spec vs remote resource.
 	b, ok := response.ResponseBody.(map[string]interface{})
 	if !ok {
 		log.Error(fmt.Errorf("body is not an object"), "Performing REST call")

internal/tools/auth/auth.go

@@ -0,0 +1,24 @@
+package auth
+
+import "fmt"
+
+type AuthType string
+
+const (
+	AuthTypeBasic  AuthType = "basic"
+	AuthTypeBearer AuthType = "bearer"
+)
+
+func (a AuthType) String() string {
+	return string(a)
+}
+
+func ToType(ty string) (AuthType, error) {
+	switch ty {
+	case "basic":
+		return AuthTypeBasic, nil
+	case "bearer":
+		return AuthTypeBearer, nil
+	}
+	return "", fmt.Errorf("unknown auth type: %s", ty)
+}

internal/tools/client/builder/builder.go

@@ -73,8 +73,13 @@ func APICallBuilder(cli restclient.UnstructuredClientInterface, info *getter.Inf
 			}
 
 			switch action {
-			case apiaction.FindBy: // FindBy is used to find the resource by the identifier fields (usually in a list of resources)
-				return cli.FindBy, callInfo, nil // FindBy has its own function
+			case apiaction.FindBy:
+				// Specialized FindBy function, we need to pass also the description in this case so we use a closure.
+				// We return a function that captures the `descr` variable from the surrounding scope
+				// and uses it when the returned function is called but still conforms to the APIFuncDef signature.
+				return func(ctx context.Context, httpClient *http.Client, path string, conf *restclient.RequestConfiguration) (restclient.Response, error) {
+					return cli.FindBy(ctx, httpClient, path, conf, &descr)
+				}, callInfo, nil
 			default:
 				return cli.Call, callInfo, nil // Generic Call function
 			}

internal/tools/client/builder/builder_test.go

@@ -56,7 +56,7 @@ func (m *mockUnstructuredClient) Call(ctx context.Context, cli *http.Client, pat
 	return restclient.Response{}, nil
 }
 
-func (m *mockUnstructuredClient) FindBy(ctx context.Context, cli *http.Client, path string, conf *restclient.RequestConfiguration) (restclient.Response, error) {
+func (m *mockUnstructuredClient) FindBy(ctx context.Context, cli *http.Client, path string, conf *restclient.RequestConfiguration, findByAction *getter.VerbsDescription) (restclient.Response, error) {
 	return restclient.Response{}, nil
 }
 

internal/tools/client/clienttools.go

@@ -14,6 +14,7 @@ import (
 
 	stringset "github.com/krateoplatformops/rest-dynamic-controller/internal/text"
 	"github.com/krateoplatformops/rest-dynamic-controller/internal/tools/comparison"
+	getter "github.com/krateoplatformops/rest-dynamic-controller/internal/tools/definitiongetter"
 	fgetter "github.com/krateoplatformops/rest-dynamic-controller/internal/tools/filegetter"
 	"github.com/pb33f/libopenapi"
 	"github.com/pb33f/libopenapi/datamodel/high/base"
@@ -31,27 +32,6 @@ import (
 
 type APICallType string
 
-type AuthType string
-
-const (
-	AuthTypeBasic  AuthType = "basic"
-	AuthTypeBearer AuthType = "bearer"
-)
-
-func (a AuthType) String() string {
-	return string(a)
-}
-
-func ToType(ty string) (AuthType, error) {
-	switch ty {
-	case "basic":
-		return AuthTypeBasic, nil
-	case "bearer":
-		return AuthTypeBearer, nil
-	}
-	return "", fmt.Errorf("unknown auth type: %s", ty)
-}
-
 func buildPath(baseUrl string, path string, parameters map[string]string, query map[string]string) *url.URL {
 	for key, param := range parameters {
 		param = url.PathEscape(param)
@@ -98,7 +78,7 @@ type UnstructuredClientInterface interface {
 	ValidateRequest(httpMethod string, path string, parameters map[string]string, query map[string]string, headers map[string]string, cookies map[string]string) error
 	RequestedBody(httpMethod string, path string) (bodys stringset.StringSet, err error)
 	RequestedParams(httpMethod string, path string) (parameters, query, headers, cookies stringset.StringSet, err error)
-	FindBy(ctx context.Context, cli *http.Client, path string, conf *RequestConfiguration) (Response, error)
+	FindBy(ctx context.Context, cli *http.Client, path string, conf *RequestConfiguration, findByAction *getter.VerbsDescription) (Response, error)
 	Call(ctx context.Context, cli *http.Client, path string, conf *RequestConfiguration) (Response, error)
 	//Validate(req *http.Request) (bool, []error) // TODO: to be re-enabled when libopenapi-validator is stable (to be renamed to ValidateRequest)
 }
@@ -127,7 +107,7 @@ type RequestConfiguration struct {
 // isInResource is a method used during a "FindBy" operation.
 // It compares a value from an API response with the corresponding value in the local Unstructured resource.
 // It checks for the identifier's presence and correctness in 'spec' first, then falls back to checking 'status'.
-// TODO: to be evaluated for potential addition of `ResponseFieldMapping` (possiblely in future versions).
+// TODO: to be evaluated for potential addition of `ResponseFieldMapping` (possibly in future versions).
 func (u *UnstructuredClient) isInResource(responseValue interface{}, fieldPath ...string) (bool, error) {
 	if u.Resource == nil {
 		return false, fmt.Errorf("resource is nil")
@@ -138,8 +118,8 @@ func (u *UnstructuredClient) isInResource(responseValue interface{}, fieldPath .
 		// If the field is found in the spec, we compare it.
 		// If it matches, we have a definitive match and can return true.
 		//log.Printf("isInResource - found in spec: localValue=%v, responseValue=%v", localValue, responseValue)
-		if comparison.CompareAny(localValue, responseValue) {
-			//log.Print("isInResource - comparison CompareAny returned true")
+		if comparison.DeepEqual(localValue, responseValue) {
+			//log.Print("isInResource - comparison DeepEqual returned true")
 			return true, nil
 		}
 	} else if err != nil {
@@ -151,7 +131,9 @@ func (u *UnstructuredClient) isInResource(responseValue interface{}, fieldPath .
 	// Last resort check, even if it makes less sense to search for findby identifiers in status.
 	if localValue, found, err := unstructured.NestedFieldNoCopy(u.Resource.Object, append([]string{"status"}, fieldPath...)...); err == nil && found {
 		// If found in status, we compare it. This is the last chance for a match.
-		if comparison.CompareAny(localValue, responseValue) {
+		//log.Printf("isInResource - found in status: localValue=%v, responseValue=%v", localValue, responseValue)
+		if comparison.DeepEqual(localValue, responseValue) {
+			//log.Print("isInResource - comparison DeepEqual returned true")
 			return true, nil
 		}
 	} else if err != nil {